README.md

@@ -1,18 +1,19 @@
 # nilmdb: Non-Intrusive Load Monitor Database
 by Jim Paris <jim@jtan.com>
 
-NilmDB requires Python 3.8 or newer.
+NilmDB requires Python 3.7 or newer.
 
 ## Prerequisites:
 
     # Runtime and build environments
-    sudo apt install python3 python3-dev python3-venv python3-pip
+    sudo apt install python3.7 python3.7-dev python3.7-venv python3-pip
 
-    # Create a new Python virtual environment to isolate deps.
-    python3 -m venv ../venv
-    source ../venv/bin/activate   # run "deactivate" to leave
+    # Optional: create a new Python virtual environment to isolate
+    # dependencies.  To leave the virtual environment, run "deactivate"
+    python -m venv venv
+    source venv/bin/activate
 
-    # Install all Python dependencies
+    # Install all Python dependencies from PyPI.
     pip3 install -r requirements.txt
 
 ## Test:
@@ -21,14 +22,6 @@ NilmDB requires Python 3.8 or newer.
 
 ## Install:
 
-Install it into the virtual environment
-
-    python3 setup.py install
-
-If you want to instead install it system-wide, you will also need to
-install the requirements system-wide:
-
-    sudo pip3 install -r requirements.txt
     sudo python3 setup.py install
 
 ## Usage:

extras/fix-oversize-files.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python
 
 import os
 import sys

nilmdb/client/client.py

@@ -145,8 +145,8 @@ class Client():
 
         Example:
           with client.stream_insert_context('/path', start, end) as ctx:
-            ctx.insert('1234567890000000 1 2 3 4\\n')
-            ctx.insert('1234567891000000 1 2 3 4\\n')
+            ctx.insert('1234567890.0 1 2 3 4\\n')
+            ctx.insert('1234567891.0 1 2 3 4\\n')
 
         For more details, see help for nilmdb.client.client.StreamInserter
 

nilmdb/client/httpclient.py

@@ -109,7 +109,7 @@ class HTTPClient():
                                           stream=False, headers=headers)
         if isjson:
             return json.loads(response.content)
-        return response.text
+        return response.content
 
     def get(self, url, params=None):
         """Simple GET (parameters in URL)"""

nilmdb/client/numpyclient.py

@@ -59,7 +59,7 @@ class NumpyClient(nilmdb.client.client.Client):
         dtype = self._get_dtype(path, layout)
 
         def to_numpy(data):
-            a = numpy.frombuffer(data, dtype)
+            a = numpy.fromstring(data, dtype)
             if structured:
                 return a
             return numpy.c_[a['timestamp'], a['data']]

nilmdb/cmdline/cmdline.py

@@ -66,7 +66,7 @@ class Complete():
         layouts = []
         for i in range(1, 10):
             layouts.extend([(t + "_" + str(i)) for t in types])
-        return (lay for lay in layouts if lay.startswith(prefix))
+        return (l for l in layouts if l.startswith(prefix))
 
     def meta_key(self, prefix, parsed_args, **kwargs):
         return (kv.split('=')[0] for kv

nilmdb/fsck/fsck.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+raise Exception("todo: fix path bytes issues")
+
 """Check database consistency, with some ability to fix problems.
 This should be able to fix cases where a database gets corrupted due
 to unexpected system shutdown, and detect other cases that may cause
@@ -11,6 +13,7 @@ import nilmdb.client.numpyclient
 from nilmdb.utils.interval import IntervalError
 from nilmdb.server.interval import Interval, IntervalSet
 from nilmdb.utils.printf import printf, fprintf, sprintf
+from nilmdb.utils.time import timestamp_to_string
 
 from collections import defaultdict
 import sqlite3
@@ -18,83 +21,70 @@ import os
 import sys
 import progressbar
 import re
+import time
 import shutil
 import pickle
 import numpy
 
-
 class FsckError(Exception):
-    def __init__(self, msg="", *args):
+    def __init__(self, msg = "", *args):
         if args:
             msg = sprintf(msg, *args)
         Exception.__init__(self, msg)
-
-
 class FixableFsckError(FsckError):
-    def __init__(self, msg=""):
-        FsckError.__init__(self, f'{msg}\nThis may be fixable with "--fix".')
-
-
+    def __init__(self, msg = "", *args):
+        if args:
+            msg = sprintf(msg, *args)
+        FsckError.__init__(self, "%s\nThis may be fixable with \"--fix\".", msg)
 class RetryFsck(FsckError):
     pass
 
-
-class FsckFormatError(FsckError):
-    pass
-
-
 def log(format, *args):
     printf(format, *args)
 
-
 def err(format, *args):
     fprintf(sys.stderr, format, *args)
 
-
 # Decorator that retries a function if it returns a specific value
-def retry_if_raised(exc, message=None, max_retries=1000):
+def retry_if_raised(exc, message = None, max_retries = 100):
     def f1(func):
         def f2(*args, **kwargs):
             for n in range(max_retries):
                 try:
                     return func(*args, **kwargs)
-                except exc:
+                except exc as e:
                     if message:
-                        log(f"{message} ({n+1})\n\n")
-            raise Exception("Max number of retries (%d) exceeded; giving up" %
-                            max_retries)
+                        log("%s\n\n", message)
+            raise Exception("Max number of retries (%d) exceeded; giving up")
         return f2
     return f1
 
-
 class Progress(object):
     def __init__(self, maxval):
         if maxval == 0:
             maxval = 1
         self.bar = progressbar.ProgressBar(
-            maxval=maxval,
-            widgets=[progressbar.Percentage(), ' ',
-                     progressbar.Bar(), ' ',
-                     progressbar.ETA()])
-        self.bar.term_width = self.bar.term_width or 75
-
+            maxval = maxval,
+            widgets = [ progressbar.Percentage(), ' ',
+                        progressbar.Bar(), ' ',
+                        progressbar.ETA() ])
+        if self.bar.term_width == 0:
+            self.bar.term_width = 75
     def __enter__(self):
         self.bar.start()
         self.last_update = 0
         return self
-
     def __exit__(self, exc_type, exc_value, traceback):
         if exc_type is None:
             self.bar.finish()
         else:
             printf("\n")
-
     def update(self, val):
         self.bar.update(val)
 
-
 class Fsck(object):
-    def __init__(self, path, fix=False):
+
+    def __init__(self, path, fix = False):
         self.basepath = path
         self.sqlpath = os.path.join(path, "data.sql")
         self.bulkpath = os.path.join(path, "data")
@@ -104,7 +94,7 @@ class Fsck(object):
     ### Main checks
 
     @retry_if_raised(RetryFsck, "Something was fixed: restarting fsck")
-    def check(self, skip_data=False):
+    def check(self, skip_data = False):
         self.bulk = None
         self.sql = None
         try:
@@ -119,9 +109,7 @@ class Fsck(object):
         finally:
             if self.bulk:
                 self.bulk.close()
-            if self.sql:  # pragma: no cover
-                # (coverage doesn't handle finally clauses correctly;
-                # both branches here are tested)
+            if self.sql:
                 self.sql.commit()
                 self.sql.close()
         log("ok\n")
@@ -176,7 +164,7 @@ class Fsck(object):
                                  "ORDER BY start_time")
             for r in result:
                 if r[0] not in self.stream_path:
-                    raise FsckError("interval ID %d not in streams", r[0])
+                    raise FsckError("interval ID %d not in streams", k)
                 self.stream_interval[r[0]].append((r[1], r[2], r[3], r[4]))
 
             log("  loading metadata\n")
@@ -184,11 +172,10 @@ class Fsck(object):
             result = cur.execute("SELECT stream_id, key, value FROM metadata")
             for r in result:
                 if r[0] not in self.stream_path:
-                    raise FsckError("metadata ID %d not in streams", r[0])
+                    raise FsckError("metadata ID %d not in streams", k)
                 if r[1] in self.stream_meta[r[0]]:
-                    raise FsckError(
-                        "duplicate metadata key '%s' for stream %d",
-                        r[1], r[0])
+                    raise FsckError("duplicate metadata key '%s' for stream %d",
+                                    r[1], r[0])
                 self.stream_meta[r[0]][r[1]] = r[2]
 
     ### Check streams and basic interval overlap
@@ -215,7 +202,6 @@ class Fsck(object):
 
                 # must exist in bulkdata
                 bulk = self.bulkpath + path
-                bulk = bulk.encode('utf-8')
                 if not os.path.isdir(bulk):
                     raise FsckError("%s: missing bulkdata dir", path)
                 if not nilmdb.server.bulkdata.Table.exists(bulk):
@@ -238,98 +224,40 @@ class Fsck(object):
                         try:
                             posiset += new
                         except IntervalError:
-                            self.fix_row_overlap(sid, path, posiset, new)
+                            raise FsckError("%s: overlap in file offsets:\n"
+                                            "set: %s\nnew: %s",
+                                            path, str(posiset), str(new))
 
+                # check bulkdata
+                self.check_bulkdata(sid, path, bulk)
+
+                # Check that we can open bulkdata
                 try:
-                    # Check bulkdata
-                    self.check_bulkdata(sid, path, bulk)
-
-                    # Check that we can open bulkdata
-                    tab = nilmdb.server.bulkdata.Table(bulk)
-                except FsckFormatError:
-                    # If there are no files except _format, try deleting
-                    # the entire stream; this may remove metadata, but
-                    # it's probably unimportant.
-                    files = list(os.listdir(bulk))
-                    if len(files) > 1:
-                        raise FsckFormatError(f"{path}: can't load _format, "
-                                              f"but data is also present")
-
-                    # Since the stream was empty, just remove it
-                    self.fix_remove_stream(sid, path, bulk,
-                                           "empty, with corrupted format file")
-                except FsckError as e:
-                    raise e
-                except Exception as e:  # pragma: no cover
-                    # No coverage because this is an unknown/unexpected error
-                    raise FsckError("%s: can't open bulkdata: %s",
-                                    path, str(e))
-                tab.close()
-
-    def fix_row_overlap(self, sid, path, existing, new):
-        # If the file rows (spos, epos) overlap in the interval table,
-        # and the overlapping ranges look like this:
-        #    A --------- C
-        #           B -------- D
-        # Then we can try changing the first interval to go from
-        # A to B instead.
-        msg = (f"{path}: overlap in file offsets:\n"
-               f"existing ranges: {existing}\n"
-               f"overlapping interval: {new}")
-        if not self.fix:
-            raise FixableFsckError(msg)
-        err(f"\n{msg}\nSeeing if we can truncate one of them...\n")
-
-        # See if there'e exactly one interval that overlaps the
-        # conflicting one in the right way
-        match = None
-        for intv in self.stream_interval[sid]:
-            (stime, etime, spos, epos) = intv
-            if spos < new.start and epos > new.start:
-                if match:
-                    err(f"no, more than one interval matched:\n"
-                        f"{intv}\n{match}\n")
-                    raise FsckError(f"{path}: unfixable overlap")
-                match = intv
-        if match is None:
-            err("no intervals overlapped in the right way\n")
-            raise FsckError(f"{path}: unfixable overlap")
-
-        # Truncate the file position
-        err(f"truncating {match}\n")
-        with self.sql:
-            cur = self.sql.cursor()
-            cur.execute("UPDATE ranges SET end_pos=? "
-                        "WHERE stream_id=? AND start_time=? AND "
-                        "end_time=? AND start_pos=? AND end_pos=?",
-                        (new.start, sid, *match))
-            if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
-                raise FsckError("failed to fix SQL database")
-        raise RetryFsck
+                    tab = None
+                    try:
+                        tab = nilmdb.server.bulkdata.Table(bulk)
+                    except Exception as e:
+                        raise FsckError("%s: can't open bulkdata: %s",
+                                        path, str(e))
+                finally:
+                    if tab:
+                        tab.close()
 
     ### Check that bulkdata is good enough to be opened
 
     @retry_if_raised(RetryFsck)
     def check_bulkdata(self, sid, path, bulk):
-        try:
-            with open(os.path.join(bulk, b"_format"), "rb") as f:
-                fmt = pickle.load(f)
-        except Exception as e:
-            raise FsckFormatError(f"{path}: can't load _format file ({e})")
-
+        with open(os.path.join(bulk, "_format"), "rb") as f:
+            fmt = pickle.load(f)
         if fmt["version"] != 3:
-            raise FsckFormatError("%s: bad or unsupported bulkdata version %d",
-                                  path, fmt["version"])
-        rows_per_file = int(fmt["rows_per_file"])
-        if rows_per_file < 1:
-            raise FsckFormatError(f"{path}: bad rows_per_file {rows_per_file}")
+            raise FsckError("%s: bad or unsupported bulkdata version %d",
+                            path, fmt["version"])
+        row_per_file = int(fmt["rows_per_file"])
         files_per_dir = int(fmt["files_per_dir"])
-        if files_per_dir < 1:
-            raise FsckFormatError(f"{path}: bad files_per_dir {files_per_dir}")
         layout = fmt["layout"]
         if layout != self.stream_layout[sid]:
-            raise FsckFormatError("%s: layout mismatch %s != %s", path,
-                                  layout, self.stream_layout[sid])
+            raise FsckError("%s: layout mismatch %s != %s", path,
+                            layout, self.stream_layout[sid])
 
         # Every file should have a size that's the multiple of the row size
         rkt = nilmdb.server.rocket.Rocket(layout, None)
@@ -337,16 +265,16 @@ class Fsck(object):
         rkt.close()
 
         # Find all directories
-        regex = re.compile(b"^[0-9a-f]{4,}$")
+        regex = re.compile("^[0-9a-f]{4,}$")
         subdirs = sorted(filter(regex.search, os.listdir(bulk)),
-                         key=lambda x: int(x, 16), reverse=True)
+                         key = lambda x: int(x, 16), reverse = True)
         for subdir in subdirs:
             # Find all files in that dir
             subpath = os.path.join(bulk, subdir)
             files = list(filter(regex.search, os.listdir(subpath)))
             if not files:
                 self.fix_empty_subdir(subpath)
-
+                raise RetryFsck
             # Verify that their size is a multiple of the row size
             for filename in files:
                 filepath = os.path.join(subpath, filename)
@@ -362,11 +290,10 @@ class Fsck(object):
         # as long as it's only ".removed" files.
         err("\n%s\n", msg)
         for fn in os.listdir(subpath):
-            if not fn.endswith(b".removed"):
+            if not fn.endswith(".removed"):
                 raise FsckError("can't fix automatically: please manually "
-                                "remove the file '%s' and try again",
-                                os.path.join(subpath, fn).decode(
-                                    'utf-8', errors='backslashreplace'))
+                                "remove the file %s and try again",
+                                os.path.join(subpath, fn))
         # Remove the whole thing
         err("Removing empty subpath\n")
         shutil.rmtree(subpath)
@@ -387,24 +314,6 @@ class Fsck(object):
             f.truncate(newsize)
             raise RetryFsck
 
-    def fix_remove_stream(self, sid, path, bulk, reason):
-        msg = f"stream {path} is corrupted: {reason}"
-        if not self.fix:
-            raise FixableFsckError(msg)
-        # Remove the stream from disk and the database
-        err(f"\n{msg}\n")
-        err(f"Removing stream {path} from disk and database\n")
-        shutil.rmtree(bulk)
-        with self.sql:
-            cur = self.sql.cursor()
-            cur.execute("DELETE FROM streams WHERE id=?",
-                        (sid,))
-            if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
-                raise FsckError("failed to remove stream")
-            cur.execute("DELETE FROM ranges WHERE stream_id=?", (sid,))
-            cur.execute("DELETE FROM metadata WHERE stream_id=?", (sid,))
-        raise RetryFsck
-
     ### Check interval endpoints
 
     def check_intervals(self):
@@ -415,12 +324,9 @@ class Fsck(object):
             for sid in self.stream_interval:
                 try:
                     bulk = self.bulkpath + self.stream_path[sid]
-                    bulk = bulk.encode('utf-8')
                     tab = nilmdb.server.bulkdata.Table(bulk)
-
                     def update(x):
                         pbar.update(done + x)
-
                     ints = self.stream_interval[sid]
                     done += self.check_table_intervals(sid, ints, tab, update)
                 finally:
@@ -429,7 +335,7 @@ class Fsck(object):
     def check_table_intervals(self, sid, ints, tab, update):
         # look in the table to make sure we can pick out the interval's
         # endpoints
-        path = self.stream_path[sid]  # noqa: F841 unused
+        path = self.stream_path[sid]
         tab.file_open.cache_remove_all()
         for (i, intv) in enumerate(ints):
             update(i)
@@ -437,11 +343,11 @@ class Fsck(object):
             if spos == epos and spos >= 0 and spos <= tab.nrows:
                 continue
             try:
-                srow = tab[spos]    # noqa: F841 unused
-                erow = tab[epos-1]  # noqa: F841 unused
+                srow = tab[spos]
+                erow = tab[epos-1]
             except Exception as e:
                 self.fix_bad_interval(sid, intv, tab, str(e))
-
+                raise RetryFsck
         return len(ints)
 
     def fix_bad_interval(self, sid, intv, tab, msg):
@@ -470,23 +376,23 @@ class Fsck(object):
                                 "end_time=? AND start_pos=? AND end_pos=?",
                                 (new_etime, new_epos, sid, stime, etime,
                                  spos, epos))
-                    if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
+                    if cur.rowcount != 1:
                         raise FsckError("failed to fix SQL database")
                 raise RetryFsck
-            err("actually it can't be truncated; times are bad too\n")
+            err("actually it can't be truncated; times are bad too")
 
         # Otherwise, the only hope is to delete the interval entirely.
         err("*** Deleting the entire interval from SQL.\n")
-        err("This may leave stale data on disk.  To fix that, copy all "
-            "data from this stream to a new stream using nilm-copy, then\n")
-        err("remove all data from and destroy %s.\n", path)
+        err("This may leave stale data on disk.  To fix that, copy all\n")
+        err("data from this stream to a new stream, then remove all data\n")
+        err("from and destroy %s.\n", path)
         with self.sql:
             cur = self.sql.cursor()
             cur.execute("DELETE FROM ranges WHERE "
                         "stream_id=? AND start_time=? AND "
                         "end_time=? AND start_pos=? AND end_pos=?",
                         (sid, stime, etime, spos, epos))
-            if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
+            if cur.rowcount != 1:
                 raise FsckError("failed to remove interval")
         raise RetryFsck
 
@@ -501,12 +407,9 @@ class Fsck(object):
             for sid in self.stream_interval:
                 try:
                     bulk = self.bulkpath + self.stream_path[sid]
-                    bulk = bulk.encode('utf-8')
                     tab = nilmdb.server.bulkdata.Table(bulk)
-
                     def update(x):
                         pbar.update(done + x)
-
                     ints = self.stream_interval[sid]
                     done += self.check_table_data(sid, ints, tab, update)
                 finally:
@@ -515,7 +418,7 @@ class Fsck(object):
     def check_table_data(self, sid, ints, tab, update):
         # Pull out all of the interval's data and verify that it's
         # monotonic.
-        maxrows = getattr(self, 'maxrows_override', 100000)
+        maxrows = 100000
         path = self.stream_path[sid]
         layout = self.stream_layout[sid]
         dtype = nilmdb.client.numpyclient.layout_to_dtype(layout)
@@ -535,76 +438,29 @@ class Fsck(object):
 
                 # Get raw data, convert to NumPy arary
                 try:
-                    raw = tab.get_data(start, stop, binary=True)
-                    data = numpy.frombuffer(raw, dtype)
-                except Exception as e:  # pragma: no cover
-                    # No coverage because it's hard to trigger this -- earlier
-                    # checks check the ranges, so this would probably be a real
-                    # disk error, malloc failure, etc.
-                    raise FsckError(
-                        "%s: failed to grab rows %d through %d: %s",
-                        path, start, stop, repr(e))
-
-                ts = data['timestamp']
-
-                # Verify that all timestamps are in range.
-                match = (ts < stime) | (ts >= etime)
-                if match.any():
-                    row = numpy.argmax(match)
-                    if ts[row] != 0:
-                        raise FsckError("%s: data timestamp %d at row %d "
-                                        "outside interval range [%d,%d)",
-                                        path, ts[row], row + start,
-                                        stime, etime)
-
-                    # Timestamp is zero and out of the expected range;
-                    # assume file ends with zeroed data and just truncate it.
-                    self.fix_table_by_truncating(
-                        path, tab, row + start,
-                        "data timestamp is out of range, and zero")
+                    raw = tab.get_data(start, stop, binary = True)
+                    data = numpy.fromstring(raw, dtype)
+                except Exception as e:
+                    raise FsckError("%s: failed to grab rows %d through %d: %s",
+                                    path, start, stop, repr(e))
 
                 # Verify that timestamps are monotonic
-                match = numpy.diff(ts) <= 0
-                if match.any():
-                    row = numpy.argmax(match)
-                    if ts[row+1] != 0:
-                        raise FsckError(
-                            "%s: non-monotonic timestamp (%d -> %d) "
-                            "at row %d", path, ts[row], ts[row+1],
-                            row + start)
-
-                    # Timestamp is zero and non-monotonic;
-                    # assume file ends with zeroed data and just truncate it.
-                    self.fix_table_by_truncating(
-                        path, tab, row + start + 1,
-                        "data timestamp is non-monotonic, and zero")
-
-                first_ts = ts[0]
+                if (numpy.diff(data['timestamp']) <= 0).any():
+                    raise FsckError("%s: non-monotonic timestamp(s) in rows "
+                                    "%d through %d", path, start, stop)
+                first_ts = data['timestamp'][0]
                 if last_ts is not None and first_ts <= last_ts:
                     raise FsckError("%s: first interval timestamp %d is not "
                                     "greater than the previous last interval "
                                     "timestamp %d, at row %d",
                                     path, first_ts, last_ts, start)
-                last_ts = ts[-1]
+                last_ts = data['timestamp'][-1]
 
-                # The previous errors are fixable, by removing the
-                # offending intervals, or changing the data
-                # timestamps.  But these are probably unlikely errors,
-                # so it's not worth implementing that yet.
+                # These are probably fixable, by removing the offending
+                # intervals.  But I'm not going to bother implementing
+                # that yet.
 
                 # Done
                 done += count
                 update(done)
         return done
-
-    def fix_table_by_truncating(self, path, tab, row, reason):
-        # Simple fix for bad data: truncate the table at the given row.
-        # On retry, fix_bad_interval will correct the database and timestamps
-        # to account for this truncation.
-        msg = f"{path}: bad data in table, starting at row {row}: {reason}"
-        if not self.fix:
-            raise FixableFsckError(msg)
-        err(f"\n{msg}\nWill try truncating table\n")
-        (subdir, fname, offs, count) = tab._offset_from_row(row)
-        tab._remove_or_truncate_file(subdir, fname, offs)
-        raise RetryFsck

nilmdb/scripts/nilmdb_fsck.py

@@ -1,27 +1,27 @@
-#!/usr/bin/env python3
+#!/usr/bin/python
 
 import nilmdb.fsck
 import argparse
-
+import os
+import sys
 
 def main():
     """Main entry point for the 'nilmdb-fsck' command line script"""
 
     parser = argparse.ArgumentParser(
-        description='Check database consistency',
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        description = 'Check database consistency',
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("-v", "--version", action="version",
-                        version=nilmdb.__version__)
+                        version = nilmdb.__version__)
     parser.add_argument("-f", "--fix", action="store_true",
-                        default=False, help='Fix errors when possible '
+                        default=False, help = 'Fix errors when possible '
                         '(which may involve removing data)')
     parser.add_argument("-n", "--no-data", action="store_true",
-                        default=False, help='Skip the slow full-data check')
-    parser.add_argument('database', help='Database directory')
+                        default=False, help = 'Skip the slow full-data check')
+    parser.add_argument('database', help = 'Database directory')
     args = parser.parse_args()
 
-    nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data=args.no_data)
-
+    nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data = args.no_data)
 
 if __name__ == "__main__":
     main()

nilmdb/scripts/nilmdb_server.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python
 
 import os
 import sys
@@ -78,12 +78,9 @@ def main():
                 stats = yappi.get_func_stats()
                 stats.sort("ttot")
                 stats.print_all()
-                try:
-                    from IPython import embed
-                    embed(header="Use the `yappi` or `stats` object to "
-                          "explore further, `quit` to exit")
-                except ModuleNotFoundError:
-                    print("\nInstall ipython to explore further")
+                from IPython import embed
+                embed(header="Use the `yappi` or `stats` object to explore "
+                      "further, quit to exit")
         else:
             server.start(blocking=True)
     except nilmdb.server.serverutil.CherryPyExit:

nilmdb/scripts/nilmtool.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python
 
 import nilmdb.cmdline
 

nilmdb/server/bulkdata.py

@@ -293,8 +293,8 @@ class Table():
             "layout": layout,
             "version": 3
         }
-        nilmdb.utils.atomic.replace_file(
-            os.path.join(root, b"_format"), pickle.dumps(fmt, 2))
+        with open(os.path.join(root, b"_format"), "wb") as f:
+            pickle.dump(fmt, f, 2)
 
     # Normal methods
     def __init__(self, root, initial_nrows=0):

nilmdb/server/rocket.c

@@ -168,7 +168,7 @@ static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
 	if (!layout)
 		return -1;
 	if (path) {
-                if (strlen(path) != (size_t)pathlen) {
+                if (strlen(path) != pathlen) {
                         PyErr_SetString(PyExc_ValueError, "path must not "
                                         "contain NUL characters");
                         return -1;
@@ -477,7 +477,7 @@ static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
         }
 
         /* Write binary data */
-        if (fwrite(data, self->binary_size, rows, self->file) != (size_t)rows) {
+        if (fwrite(data, self->binary_size, rows, self->file) != rows) {
                 PyErr_SetFromErrno(PyExc_OSError);
                 return NULL;
         }
@@ -628,7 +628,7 @@ static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args)
 
         /* Data in the file is already in the desired little-endian
            binary format, so just read it directly. */
-        if (fread(str, self->binary_size, count, self->file) != (size_t)count) {
+        if (fread(str, self->binary_size, count, self->file) != count) {
                 free(str);
                 PyErr_SetFromErrno(PyExc_OSError);
                 return NULL;

nilmdb/server/serverutil.py

@@ -6,7 +6,6 @@ import sys
 import json
 import decorator
 import functools
-import threading
 
 import cherrypy
 
@@ -179,19 +178,6 @@ def cherrypy_patch_exit():
             os._exit = real_exit
     bus.exit = functools.partial(patched_exit, bus.exit)
 
-    # A behavior change in Python 3.8 means that some thread exceptions,
-    # derived from SystemExit, now print tracebacks where they didn't
-    # used to: https://bugs.python.org/issue1230540
-    # Install a thread exception hook that ignores CherryPyExit;
-    # to make this match the behavior where we didn't set
-    # threading.excepthook, we also need to ignore SystemExit.
-    def hook(args):
-        if args.exc_type == CherryPyExit or args.exc_type == SystemExit:
-            return
-        sys.excepthook(args.exc_type, args.exc_value,
-                       args.exc_traceback)  # pragma: no cover
-    threading.excepthook = hook
-
 
 # Start/stop CherryPy standalone server
 def cherrypy_start(blocking=False, event=False):

nilmdb/utils/timestamper.py

@@ -26,9 +26,7 @@ class Timestamper():
                 return b""
             if line[0:1] == b'#':
                 continue
-            # For some reason, coverage on python 3.8 reports that
-            # we never hit this break, even though we definitely do.
-            break  # pragma: no cover
+            break
         try:
             return next(self.ts_iter) + line
         except StopIteration:

requirements.txt

@@ -1,41 +1,16 @@
-argcomplete==1.12.0
-CherryPy==18.6.0
-coverage==5.2.1
-Cython==0.29.21
-decorator==4.4.2
-fallocate==1.6.4
-flake8==3.8.3
-nose==1.3.7
-numpy==1.19.1
-progressbar==2.5
-psutil==5.7.2
-python-datetime-tz==0.5.4
-python-dateutil==2.8.1
-requests==2.24.0
-tz==0.2.2
-yappi==1.2.5
-
-## The following requirements were added by pip freeze:
-beautifulsoup4==4.9.1
-certifi==2020.6.20
-chardet==3.0.4
-cheroot==8.4.2
-idna==2.10
-jaraco.classes==3.1.0
-jaraco.collections==3.0.0
-jaraco.functools==3.0.1
-jaraco.text==3.2.0
-mccabe==0.6.1
-more-itertools==8.4.0
-portend==2.6
-pycodestyle==2.6.0
-pyflakes==2.2.0
-pytz==2020.1
-six==1.15.0
-soupsieve==2.0.1
-tempora==4.0.0
-urllib3==1.25.10
-waitress==1.4.4
-WebOb==1.8.6
-WebTest==2.0.35
-zc.lockfile==2.0
+argcomplete>=1.10.0
+CherryPy>=18.1.2
+coverage>=4.5.4
+cython>=0.29.13
+decorator>=4.4.0
+fallocate>=1.6.4
+flake8>=3.7.8
+nose>=1.3.7
+numpy>=1.17.0
+progressbar>=2.5
+psutil>=5.6.3
+python-datetime-tz>=0.5.4
+python-dateutil>=2.8.0
+requests>=2.22.0
+tz>=0.2.2
+WebTest>=2.0.33

setup.cfg

@@ -47,13 +47,10 @@ tag_prefix=nilmdb-
 parentdir_prefix=nilmdb-
 
 [flake8]
-exclude=_version.py
+exclude=_version.py,fsck.py,nilmdb_fsck.py
 extend-ignore=E731
-per-file-ignores=__init__.py:F401,E402 \
-        serializer.py:E722 \
-        mustclose.py:E722 \
-        fsck.py:E266
+per-file-ignores=__init__.py:F401,E402 serializer.py:E722 mustclose.py:E722
 
 [pylint]
-ignore=_version.py
+ignore=_version.py,fsck.py,nilmdb_fsck.py
 disable=C0103,C0111,R0913,R0914

setup.py

@@ -1,10 +1,10 @@
-#!/usr/bin/env python3
+#!/usr/bin/python
 
 # To release a new version, tag it:
 #   git tag -a nilmdb-1.1 -m "Version 1.1"
 #   git push --tags
 # Then just package it up:
-#   python3 setup.py sdist
+#   python setup.py sdist
 
 import sys
 import os
@@ -36,7 +36,7 @@ install_requires = open('requirements.txt').readlines()
 setup(name='nilmdb',
       version = versioneer.get_version(),
       cmdclass = versioneer.get_cmdclass(),
-      url = 'https://git.jim.sh/nilm/nilmdb.git',
+      url = 'https://git.jim.sh/jim/lees/nilmdb.git',
       author = 'Jim Paris',
       description = "NILM Database",
       long_description = "NILM Database",

tests/fsck-data/test1/data/git-empty-dir-placeholder

@@ -1 +0,0 @@
-hi

tests/fsck-data/test1a/data/git-empty-dir-placeholder

@@ -1 +0,0 @@
-hi

tests/fsck-data/test1c/data/git-empty-dir-placeholder

@@ -1 +0,0 @@
-hi

tests/fsck-data/test2h/data/git-empty-dir-placeholder

@@ -1 +0,0 @@
-hi

tests/fsck-data/test2p/data/a/b/0001/git-empty-dir-placeholder

@@ -1 +0,0 @@
-hi

tests/fsck-data/test2p1/data/a/b/0001/hello

@@ -1 +0,0 @@
-world

tests/fsck-data/test2p2/data/a/b/0001/hello.removed

@@ -1 +0,0 @@
-world