Start implementing fsck test and porting fsck to Python 3

nilmdb/fsck/fsck.py

@@ -211,6 +211,7 @@ class Fsck(object):
 
                 # must exist in bulkdata
                 bulk = self.bulkpath + path
+                bulk = bulk.encode('utf-8')
                 if not os.path.isdir(bulk):
                     raise FsckError("%s: missing bulkdata dir", path)
                 if not nilmdb.server.bulkdata.Table.exists(bulk):
@@ -256,7 +257,7 @@ class Fsck(object):
 
     @retry_if_raised(RetryFsck)
     def check_bulkdata(self, sid, path, bulk):
-        with open(os.path.join(bulk, "_format"), "rb") as f:
+        with open(os.path.join(bulk, b"_format"), "rb") as f:
             fmt = pickle.load(f)
         if fmt["version"] != 3:
             raise FsckError("%s: bad or unsupported bulkdata version %d",
@@ -278,7 +279,7 @@ class Fsck(object):
         rkt.close()
 
         # Find all directories
-        regex = re.compile("^[0-9a-f]{4,}$")
+        regex = re.compile(b"^[0-9a-f]{4,}$")
         subdirs = sorted(filter(regex.search, os.listdir(bulk)),
                          key=lambda x: int(x, 16), reverse=True)
         for subdir in subdirs:
@@ -337,6 +338,7 @@ class Fsck(object):
             for sid in self.stream_interval:
                 try:
                     bulk = self.bulkpath + self.stream_path[sid]
+                    bulk = bulk.encode('utf-8')
                     tab = nilmdb.server.bulkdata.Table(bulk)
 
                     def update(x):
@@ -422,6 +424,7 @@ class Fsck(object):
             for sid in self.stream_interval:
                 try:
                     bulk = self.bulkpath + self.stream_path[sid]
+                    bulk = bulk.encode('utf-8')
                     tab = nilmdb.server.bulkdata.Table(bulk)
 
                     def update(x):

tests/test_fsck.py

@@ -0,0 +1,118 @@
+import nilmdb
+import nilmdb.fsck
+
+from nilmdb.utils.printf import printf
+from nose.tools import assert_raises
+
+import io
+import sys
+import shutil
+import traceback
+from testutil.helpers import *
+
+class TestFsck(object):
+
+    def run(self, db, fix=True, skip=False, checker=None):
+        """Run fsck client with the given test database.  Save the output and
+        exit code.
+        """
+        if db is not None:
+            recursive_unlink("tests/fsck-testdb")
+            shutil.copytree(f"tests/fsck-data/{db}", "tests/fsck-testdb")
+        class stdio_wrapper:
+            def __init__(self, stdin, stdout, stderr):
+                self.io = (stdin, stdout, stderr)
+            def __enter__(self):
+                self.saved = ( sys.stdin, sys.stdout, sys.stderr )
+                ( sys.stdin, sys.stdout, sys.stderr ) = self.io
+            def __exit__(self, type, value, traceback):
+                ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
+        # Empty input
+        infile = io.TextIOWrapper(io.BytesIO(b""))
+        # Capture stdout, stderr
+        errfile = io.TextIOWrapper(io.BytesIO())
+        outfile = errfile
+        with stdio_wrapper(infile, outfile, errfile) as s:
+            try:
+                f = nilmdb.fsck.Fsck("tests/fsck-testdb", fix)
+                if checker:
+                    checker(f)
+                else:
+                    f.check(skip_data=skip)
+                sys.exit(0)
+            except SystemExit as e:
+                exitcode = e.code
+            except Exception as e:
+                traceback.print_exc()
+                exitcode = 1
+
+        # Capture raw binary output, and also try to decode a Unicode
+        # string copy.
+        self.captured_binary = outfile.buffer.getvalue()
+        try:
+            outfile.seek(0)
+            self.captured = outfile.read()
+        except UnicodeDecodeError:
+            self.captured = None
+
+        self.exitcode = exitcode
+
+    def ok(self, *args, **kwargs):
+        self.run(*args, **kwargs)
+        if self.exitcode != 0:
+            self.dump()
+            eq_(self.exitcode, 0)
+
+    def okmsg(self, db, expect):
+        self.ok(db)
+        self.contain(expect)
+
+    def fail(self, *args, exitcode=None, **kwargs):
+        self.run(*args, **kwargs)
+        if exitcode is not None and self.exitcode != exitcode:
+            # Wrong exit code
+            self.dump()
+            eq_(self.exitcode, exitcode)
+        if self.exitcode == 0:
+            # Success, when we wanted failure
+            self.dump()
+            ne_(self.exitcode, 0)
+
+    def failmsg(self, db, expect):
+        self.fail(db)
+        self.contain(expect)
+
+    def contain(self, checkstring, contain=True):
+        if contain:
+            in_(checkstring, self.captured)
+        else:
+            nin_(checkstring, self.captured)
+
+    def dump(self):
+        printf("\n===dump start===\n%s===dump end===\n", self.captured)
+
+
+    def test_fsck(self):
+        self.okmsg("test1", "\nok")
+
+        def check_paths_twice(f):
+            f.check()
+            f.check_paths()
+            f.bulk.close()
+        self.ok("test1", checker=check_paths_twice)
+
+        with open("tests/fsck-testdb/data.lock", "w") as lock:
+            nilmdb.utils.lock.exclusive_lock(lock)
+            self.failmsg(None, "Database already locked")
+
+        self.failmsg("test1a", "SQL database missing")
+        self.failmsg("test1b", "Bulk data directory missing")
+        self.failmsg("test1c", "database version 0 too old")
+
+        self.okmsg("test2", "\nok")
+
+        self.failmsg("test2a", "duplicated ID 1 in stream IDs")
+        self.failmsg("test2b", "interval ID 2 not in streams")
+        self.failmsg("test2c", "metadata ID 2 not in streams")
+        self.failmsg("test2d", "duplicate metadata key")
+