diff --git a/nilmdb/fsck/fsck.py b/nilmdb/fsck/fsck.py index a40c327..8b38a1b 100644 --- a/nilmdb/fsck/fsck.py +++ b/nilmdb/fsck/fsck.py @@ -211,6 +211,7 @@ class Fsck(object): # must exist in bulkdata bulk = self.bulkpath + path + bulk = bulk.encode('utf-8') if not os.path.isdir(bulk): raise FsckError("%s: missing bulkdata dir", path) if not nilmdb.server.bulkdata.Table.exists(bulk): @@ -256,7 +257,7 @@ class Fsck(object): @retry_if_raised(RetryFsck) def check_bulkdata(self, sid, path, bulk): - with open(os.path.join(bulk, "_format"), "rb") as f: + with open(os.path.join(bulk, b"_format"), "rb") as f: fmt = pickle.load(f) if fmt["version"] != 3: raise FsckError("%s: bad or unsupported bulkdata version %d", @@ -278,7 +279,7 @@ class Fsck(object): rkt.close() # Find all directories - regex = re.compile("^[0-9a-f]{4,}$") + regex = re.compile(b"^[0-9a-f]{4,}$") subdirs = sorted(filter(regex.search, os.listdir(bulk)), key=lambda x: int(x, 16), reverse=True) for subdir in subdirs: @@ -337,6 +338,7 @@ class Fsck(object): for sid in self.stream_interval: try: bulk = self.bulkpath + self.stream_path[sid] + bulk = bulk.encode('utf-8') tab = nilmdb.server.bulkdata.Table(bulk) def update(x): @@ -422,6 +424,7 @@ class Fsck(object): for sid in self.stream_interval: try: bulk = self.bulkpath + self.stream_path[sid] + bulk = bulk.encode('utf-8') tab = nilmdb.server.bulkdata.Table(bulk) def update(x): diff --git a/tests/fsck-data/test1/data.sql b/tests/fsck-data/test1/data.sql new file mode 100644 index 0000000..aa1759e Binary files /dev/null and b/tests/fsck-data/test1/data.sql differ diff --git a/tests/fsck-data/test1b/data.sql b/tests/fsck-data/test1b/data.sql new file mode 100644 index 0000000..aa1759e Binary files /dev/null and b/tests/fsck-data/test1b/data.sql differ diff --git a/tests/fsck-data/test1c/data.sql b/tests/fsck-data/test1c/data.sql new file mode 100644 index 0000000..36fdf42 Binary files /dev/null and b/tests/fsck-data/test1c/data.sql differ diff --git a/tests/fsck-data/test2/data.sql b/tests/fsck-data/test2/data.sql new file mode 100644 index 0000000..21c3f35 Binary files /dev/null and b/tests/fsck-data/test2/data.sql differ diff --git a/tests/fsck-data/test2/data/a/b/0000/0000 b/tests/fsck-data/test2/data/a/b/0000/0000 new file mode 100644 index 0000000..c0605f5 Binary files /dev/null and b/tests/fsck-data/test2/data/a/b/0000/0000 differ diff --git a/tests/fsck-data/test2/data/a/b/_format b/tests/fsck-data/test2/data/a/b/_format new file mode 100644 index 0000000..ca49cc2 Binary files /dev/null and b/tests/fsck-data/test2/data/a/b/_format differ diff --git a/tests/fsck-data/test2a/data.sql b/tests/fsck-data/test2a/data.sql new file mode 100644 index 0000000..ecee5ec Binary files /dev/null and b/tests/fsck-data/test2a/data.sql differ diff --git a/tests/fsck-data/test2a/data/a/b/0000/0000 b/tests/fsck-data/test2a/data/a/b/0000/0000 new file mode 100644 index 0000000..c0605f5 Binary files /dev/null and b/tests/fsck-data/test2a/data/a/b/0000/0000 differ diff --git a/tests/fsck-data/test2a/data/a/b/_format b/tests/fsck-data/test2a/data/a/b/_format new file mode 100644 index 0000000..ca49cc2 Binary files /dev/null and b/tests/fsck-data/test2a/data/a/b/_format differ diff --git a/tests/fsck-data/test2b/data.sql b/tests/fsck-data/test2b/data.sql new file mode 100644 index 0000000..0af2acc Binary files /dev/null and b/tests/fsck-data/test2b/data.sql differ diff --git a/tests/fsck-data/test2b/data/a/b/0000/0000 b/tests/fsck-data/test2b/data/a/b/0000/0000 new file mode 100644 index 0000000..c0605f5 Binary files /dev/null and b/tests/fsck-data/test2b/data/a/b/0000/0000 differ diff --git a/tests/fsck-data/test2b/data/a/b/_format b/tests/fsck-data/test2b/data/a/b/_format new file mode 100644 index 0000000..ca49cc2 Binary files /dev/null and b/tests/fsck-data/test2b/data/a/b/_format differ diff --git a/tests/fsck-data/test2c/data.sql b/tests/fsck-data/test2c/data.sql new file mode 100644 index 0000000..0df1b65 Binary files /dev/null and b/tests/fsck-data/test2c/data.sql differ diff --git a/tests/fsck-data/test2c/data/a/b/0000/0000 b/tests/fsck-data/test2c/data/a/b/0000/0000 new file mode 100644 index 0000000..c0605f5 Binary files /dev/null and b/tests/fsck-data/test2c/data/a/b/0000/0000 differ diff --git a/tests/fsck-data/test2c/data/a/b/_format b/tests/fsck-data/test2c/data/a/b/_format new file mode 100644 index 0000000..ca49cc2 Binary files /dev/null and b/tests/fsck-data/test2c/data/a/b/_format differ diff --git a/tests/fsck-data/test2d/data.sql b/tests/fsck-data/test2d/data.sql new file mode 100644 index 0000000..d059842 Binary files /dev/null and b/tests/fsck-data/test2d/data.sql differ diff --git a/tests/fsck-data/test2d/data/a/b/0000/0000 b/tests/fsck-data/test2d/data/a/b/0000/0000 new file mode 100644 index 0000000..c0605f5 Binary files /dev/null and b/tests/fsck-data/test2d/data/a/b/0000/0000 differ diff --git a/tests/fsck-data/test2d/data/a/b/_format b/tests/fsck-data/test2d/data/a/b/_format new file mode 100644 index 0000000..ca49cc2 Binary files /dev/null and b/tests/fsck-data/test2d/data/a/b/_format differ diff --git a/tests/test_fsck.py b/tests/test_fsck.py new file mode 100644 index 0000000..ab05b46 --- /dev/null +++ b/tests/test_fsck.py @@ -0,0 +1,118 @@ +import nilmdb +import nilmdb.fsck + +from nilmdb.utils.printf import printf +from nose.tools import assert_raises + +import io +import sys +import shutil +import traceback +from testutil.helpers import * + +class TestFsck(object): + + def run(self, db, fix=True, skip=False, checker=None): + """Run fsck client with the given test database. Save the output and + exit code. + """ + if db is not None: + recursive_unlink("tests/fsck-testdb") + shutil.copytree(f"tests/fsck-data/{db}", "tests/fsck-testdb") + class stdio_wrapper: + def __init__(self, stdin, stdout, stderr): + self.io = (stdin, stdout, stderr) + def __enter__(self): + self.saved = ( sys.stdin, sys.stdout, sys.stderr ) + ( sys.stdin, sys.stdout, sys.stderr ) = self.io + def __exit__(self, type, value, traceback): + ( sys.stdin, sys.stdout, sys.stderr ) = self.saved + # Empty input + infile = io.TextIOWrapper(io.BytesIO(b"")) + # Capture stdout, stderr + errfile = io.TextIOWrapper(io.BytesIO()) + outfile = errfile + with stdio_wrapper(infile, outfile, errfile) as s: + try: + f = nilmdb.fsck.Fsck("tests/fsck-testdb", fix) + if checker: + checker(f) + else: + f.check(skip_data=skip) + sys.exit(0) + except SystemExit as e: + exitcode = e.code + except Exception as e: + traceback.print_exc() + exitcode = 1 + + # Capture raw binary output, and also try to decode a Unicode + # string copy. + self.captured_binary = outfile.buffer.getvalue() + try: + outfile.seek(0) + self.captured = outfile.read() + except UnicodeDecodeError: + self.captured = None + + self.exitcode = exitcode + + def ok(self, *args, **kwargs): + self.run(*args, **kwargs) + if self.exitcode != 0: + self.dump() + eq_(self.exitcode, 0) + + def okmsg(self, db, expect): + self.ok(db) + self.contain(expect) + + def fail(self, *args, exitcode=None, **kwargs): + self.run(*args, **kwargs) + if exitcode is not None and self.exitcode != exitcode: + # Wrong exit code + self.dump() + eq_(self.exitcode, exitcode) + if self.exitcode == 0: + # Success, when we wanted failure + self.dump() + ne_(self.exitcode, 0) + + def failmsg(self, db, expect): + self.fail(db) + self.contain(expect) + + def contain(self, checkstring, contain=True): + if contain: + in_(checkstring, self.captured) + else: + nin_(checkstring, self.captured) + + def dump(self): + printf("\n===dump start===\n%s===dump end===\n", self.captured) + + + def test_fsck(self): + self.okmsg("test1", "\nok") + + def check_paths_twice(f): + f.check() + f.check_paths() + f.bulk.close() + self.ok("test1", checker=check_paths_twice) + + with open("tests/fsck-testdb/data.lock", "w") as lock: + nilmdb.utils.lock.exclusive_lock(lock) + self.failmsg(None, "Database already locked") + + self.failmsg("test1a", "SQL database missing") + self.failmsg("test1b", "Bulk data directory missing") + self.failmsg("test1c", "database version 0 too old") + + self.okmsg("test2", "\nok") + + self.failmsg("test2a", "duplicated ID 1 in stream IDs") + self.failmsg("test2b", "interval ID 2 not in streams") + self.failmsg("test2c", "metadata ID 2 not in streams") + self.failmsg("test2d", "duplicate metadata key") +