Browse Source

Start implementing fsck test and porting fsck to Python 3

tags/nilmdb-2.1.0
Jim Paris 1 year ago
parent
commit
99ac47cf0d
20 changed files with 123 additions and 2 deletions
  1. +5
    -2
      nilmdb/fsck/fsck.py
  2. BIN
      tests/fsck-data/test1/data.sql
  3. BIN
      tests/fsck-data/test1b/data.sql
  4. BIN
      tests/fsck-data/test1c/data.sql
  5. BIN
      tests/fsck-data/test2/data.sql
  6. BIN
      tests/fsck-data/test2/data/a/b/0000/0000
  7. BIN
      tests/fsck-data/test2/data/a/b/_format
  8. BIN
      tests/fsck-data/test2a/data.sql
  9. BIN
      tests/fsck-data/test2a/data/a/b/0000/0000
  10. BIN
      tests/fsck-data/test2a/data/a/b/_format
  11. BIN
      tests/fsck-data/test2b/data.sql
  12. BIN
      tests/fsck-data/test2b/data/a/b/0000/0000
  13. BIN
      tests/fsck-data/test2b/data/a/b/_format
  14. BIN
      tests/fsck-data/test2c/data.sql
  15. BIN
      tests/fsck-data/test2c/data/a/b/0000/0000
  16. BIN
      tests/fsck-data/test2c/data/a/b/_format
  17. BIN
      tests/fsck-data/test2d/data.sql
  18. BIN
      tests/fsck-data/test2d/data/a/b/0000/0000
  19. BIN
      tests/fsck-data/test2d/data/a/b/_format
  20. +118
    -0
      tests/test_fsck.py

+ 5
- 2
nilmdb/fsck/fsck.py View File

@@ -211,6 +211,7 @@ class Fsck(object):

# must exist in bulkdata
bulk = self.bulkpath + path
bulk = bulk.encode('utf-8')
if not os.path.isdir(bulk):
raise FsckError("%s: missing bulkdata dir", path)
if not nilmdb.server.bulkdata.Table.exists(bulk):
@@ -256,7 +257,7 @@ class Fsck(object):

@retry_if_raised(RetryFsck)
def check_bulkdata(self, sid, path, bulk):
with open(os.path.join(bulk, "_format"), "rb") as f:
with open(os.path.join(bulk, b"_format"), "rb") as f:
fmt = pickle.load(f)
if fmt["version"] != 3:
raise FsckError("%s: bad or unsupported bulkdata version %d",
@@ -278,7 +279,7 @@ class Fsck(object):
rkt.close()

# Find all directories
regex = re.compile("^[0-9a-f]{4,}$")
regex = re.compile(b"^[0-9a-f]{4,}$")
subdirs = sorted(filter(regex.search, os.listdir(bulk)),
key=lambda x: int(x, 16), reverse=True)
for subdir in subdirs:
@@ -337,6 +338,7 @@ class Fsck(object):
for sid in self.stream_interval:
try:
bulk = self.bulkpath + self.stream_path[sid]
bulk = bulk.encode('utf-8')
tab = nilmdb.server.bulkdata.Table(bulk)

def update(x):
@@ -422,6 +424,7 @@ class Fsck(object):
for sid in self.stream_interval:
try:
bulk = self.bulkpath + self.stream_path[sid]
bulk = bulk.encode('utf-8')
tab = nilmdb.server.bulkdata.Table(bulk)

def update(x):


BIN
tests/fsck-data/test1/data.sql View File


BIN
tests/fsck-data/test1b/data.sql View File


BIN
tests/fsck-data/test1c/data.sql View File


BIN
tests/fsck-data/test2/data.sql View File


BIN
tests/fsck-data/test2/data/a/b/0000/0000 View File


BIN
tests/fsck-data/test2/data/a/b/_format View File


BIN
tests/fsck-data/test2a/data.sql View File


BIN
tests/fsck-data/test2a/data/a/b/0000/0000 View File


BIN
tests/fsck-data/test2a/data/a/b/_format View File


BIN
tests/fsck-data/test2b/data.sql View File


BIN
tests/fsck-data/test2b/data/a/b/0000/0000 View File


BIN
tests/fsck-data/test2b/data/a/b/_format View File


BIN
tests/fsck-data/test2c/data.sql View File


BIN
tests/fsck-data/test2c/data/a/b/0000/0000 View File


BIN
tests/fsck-data/test2c/data/a/b/_format View File


BIN
tests/fsck-data/test2d/data.sql View File


BIN
tests/fsck-data/test2d/data/a/b/0000/0000 View File


BIN
tests/fsck-data/test2d/data/a/b/_format View File


+ 118
- 0
tests/test_fsck.py View File

@@ -0,0 +1,118 @@
import nilmdb
import nilmdb.fsck

from nilmdb.utils.printf import printf
from nose.tools import assert_raises

import io
import sys
import shutil
import traceback
from testutil.helpers import *

class TestFsck(object):

def run(self, db, fix=True, skip=False, checker=None):
"""Run fsck client with the given test database. Save the output and
exit code.
"""
if db is not None:
recursive_unlink("tests/fsck-testdb")
shutil.copytree(f"tests/fsck-data/{db}", "tests/fsck-testdb")
class stdio_wrapper:
def __init__(self, stdin, stdout, stderr):
self.io = (stdin, stdout, stderr)
def __enter__(self):
self.saved = ( sys.stdin, sys.stdout, sys.stderr )
( sys.stdin, sys.stdout, sys.stderr ) = self.io
def __exit__(self, type, value, traceback):
( sys.stdin, sys.stdout, sys.stderr ) = self.saved
# Empty input
infile = io.TextIOWrapper(io.BytesIO(b""))
# Capture stdout, stderr
errfile = io.TextIOWrapper(io.BytesIO())
outfile = errfile
with stdio_wrapper(infile, outfile, errfile) as s:
try:
f = nilmdb.fsck.Fsck("tests/fsck-testdb", fix)
if checker:
checker(f)
else:
f.check(skip_data=skip)
sys.exit(0)
except SystemExit as e:
exitcode = e.code
except Exception as e:
traceback.print_exc()
exitcode = 1

# Capture raw binary output, and also try to decode a Unicode
# string copy.
self.captured_binary = outfile.buffer.getvalue()
try:
outfile.seek(0)
self.captured = outfile.read()
except UnicodeDecodeError:
self.captured = None

self.exitcode = exitcode

def ok(self, *args, **kwargs):
self.run(*args, **kwargs)
if self.exitcode != 0:
self.dump()
eq_(self.exitcode, 0)

def okmsg(self, db, expect):
self.ok(db)
self.contain(expect)

def fail(self, *args, exitcode=None, **kwargs):
self.run(*args, **kwargs)
if exitcode is not None and self.exitcode != exitcode:
# Wrong exit code
self.dump()
eq_(self.exitcode, exitcode)
if self.exitcode == 0:
# Success, when we wanted failure
self.dump()
ne_(self.exitcode, 0)

def failmsg(self, db, expect):
self.fail(db)
self.contain(expect)

def contain(self, checkstring, contain=True):
if contain:
in_(checkstring, self.captured)
else:
nin_(checkstring, self.captured)

def dump(self):
printf("\n===dump start===\n%s===dump end===\n", self.captured)


def test_fsck(self):
self.okmsg("test1", "\nok")

def check_paths_twice(f):
f.check()
f.check_paths()
f.bulk.close()
self.ok("test1", checker=check_paths_twice)

with open("tests/fsck-testdb/data.lock", "w") as lock:
nilmdb.utils.lock.exclusive_lock(lock)
self.failmsg(None, "Database already locked")

self.failmsg("test1a", "SQL database missing")
self.failmsg("test1b", "Bulk data directory missing")
self.failmsg("test1c", "database version 0 too old")

self.okmsg("test2", "\nok")

self.failmsg("test2a", "duplicated ID 1 in stream IDs")
self.failmsg("test2b", "interval ID 2 not in streams")
self.failmsg("test2c", "metadata ID 2 not in streams")
self.failmsg("test2d", "duplicate metadata key")


Loading…
Cancel
Save