You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

test_fsck.py 6.4 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. import nilmdb
  2. import nilmdb.fsck
  3. from nilmdb.utils.printf import printf
  4. from nose.tools import assert_raises
  5. import io
  6. import sys
  7. import shutil
  8. import traceback
  9. from testutil.helpers import *
  10. class TestFsck(object):
  11. def run(self, db, fix=True, skip=False, checker=None):
  12. """Run fsck client with the given test database. Save the output and
  13. exit code.
  14. """
  15. if db is not None:
  16. recursive_unlink("tests/fsck-testdb")
  17. shutil.copytree(f"tests/fsck-data/{db}", "tests/fsck-testdb",
  18. ignore=shutil.ignore_patterns(
  19. "git-empty-dir-placeholder"))
  20. class stdio_wrapper:
  21. def __init__(self, stdin, stdout, stderr):
  22. self.io = (stdin, stdout, stderr)
  23. def __enter__(self):
  24. self.saved = ( sys.stdin, sys.stdout, sys.stderr )
  25. ( sys.stdin, sys.stdout, sys.stderr ) = self.io
  26. def __exit__(self, type, value, traceback):
  27. ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
  28. # Empty input
  29. infile = io.TextIOWrapper(io.BytesIO(b""))
  30. # Capture stdout, stderr
  31. errfile = io.TextIOWrapper(io.BytesIO())
  32. outfile = errfile
  33. with stdio_wrapper(infile, outfile, errfile) as s:
  34. try:
  35. f = nilmdb.fsck.Fsck("tests/fsck-testdb", fix)
  36. if checker:
  37. checker(f)
  38. else:
  39. f.check(skip_data=skip)
  40. sys.exit(0)
  41. except SystemExit as e:
  42. exitcode = e.code
  43. except Exception as e:
  44. traceback.print_exc()
  45. exitcode = 1
  46. # Capture raw binary output, and also try to decode a Unicode
  47. # string copy.
  48. self.captured_binary = outfile.buffer.getvalue()
  49. try:
  50. outfile.seek(0)
  51. self.captured = outfile.read()
  52. except UnicodeDecodeError:
  53. self.captured = None
  54. self.exitcode = exitcode
  55. def ok(self, *args, **kwargs):
  56. self.run(*args, **kwargs)
  57. if self.exitcode != 0:
  58. self.dump()
  59. eq_(self.exitcode, 0)
  60. def okmsg(self, db, expect, **kwargs):
  61. self.ok(db, **kwargs)
  62. self.contain(expect)
  63. def fail(self, *args, exitcode=None, **kwargs):
  64. self.run(*args, **kwargs)
  65. if exitcode is not None and self.exitcode != exitcode:
  66. # Wrong exit code
  67. self.dump()
  68. eq_(self.exitcode, exitcode)
  69. if self.exitcode == 0:
  70. # Success, when we wanted failure
  71. self.dump()
  72. ne_(self.exitcode, 0)
  73. def failmsg(self, db, expect, **kwargs):
  74. self.fail(db, **kwargs)
  75. self.contain(expect)
  76. def contain(self, checkstring, contain=True):
  77. if contain:
  78. in_(checkstring, self.captured)
  79. else:
  80. nin_(checkstring, self.captured)
  81. def dump(self):
  82. printf("\n===dump start===\n%s===dump end===\n", self.captured)
  83. def test_fsck(self):
  84. self.okmsg("test1", "\nok")
  85. def check_paths_twice(f):
  86. f.check()
  87. f.check_paths()
  88. f.bulk.close()
  89. self.ok("test1", checker=check_paths_twice)
  90. with open("tests/fsck-testdb/data.lock", "w") as lock:
  91. nilmdb.utils.lock.exclusive_lock(lock)
  92. self.failmsg(None, "Database already locked")
  93. self.failmsg("test1a", "SQL database missing")
  94. self.failmsg("test1b", "Bulk data directory missing")
  95. self.failmsg("test1c", "database version 0 too old")
  96. self.okmsg("test2", "\nok")
  97. self.failmsg("test2a", "duplicated ID 1 in stream IDs")
  98. self.failmsg("test2b", "interval ID 2 not in streams")
  99. self.failmsg("test2c", "metadata ID 2 not in streams")
  100. self.failmsg("test2d", "duplicate metadata key")
  101. self.failmsg("test2e", "duplicated path")
  102. self.failmsg("test2f", "bad layout")
  103. self.failmsg("test2g", "bad count")
  104. self.failmsg("test2h", "missing bulkdata dir")
  105. self.failmsg("test2i", "bad bulkdata table")
  106. self.failmsg("test2j", "overlap in intervals")
  107. self.failmsg("test2k", "overlap in file offsets")
  108. self.ok("test2k1")
  109. self.failmsg("test2l", "unsupported bulkdata version")
  110. self.failmsg("test2m", "bad rows_per_file")
  111. self.failmsg("test2n", "bad files_per_dir")
  112. self.failmsg("test2o", "layout mismatch")
  113. self.failmsg("test2p", "missing data files", fix=False)
  114. self.contain("This may be fixable")
  115. self.okmsg("test2p", "Removing empty subpath")
  116. self.failmsg("test2p1", "please manually remove the file")
  117. self.okmsg("test2p2", "Removing empty subpath")
  118. self.failmsg("test2q", "extra bytes present", fix=False)
  119. self.okmsg("test2q", "Truncating file")
  120. self.failmsg("test2r", "error accessing rows", fix=False)
  121. self.okmsg("test2r", "end position is past endrows")
  122. self.okmsg("test2r1", "actually it can't be truncated")
  123. self.contain("Deleting the entire interval")
  124. self.contain("restarting fsck")
  125. self.ok("test2r2")
  126. self.failmsg("test2s", "non-monotonic timestamp (1000000 -> 12345)")
  127. def check_small_maxrows(f):
  128. f.maxrows_override = 1
  129. f.check()
  130. self.fail("test2t", checker=check_small_maxrows)
  131. self.contain("first interval timestamp 0 is not greater")
  132. self.ok("test2t", skip=True)
  133. self.failmsg("test2u", "data timestamp 1234567890 at row 28 outside")
  134. self.failmsg("test2u1", "data timestamp 7 at row 0 outside")
  135. @nilmdb.fsck.fsck.retry_if_raised(Exception, max_retries=3)
  136. def foo():
  137. raise Exception("hi")
  138. with assert_raises(Exception):
  139. foo()
  140. self.failmsg("test2v", "can't load _format, but data is also present")
  141. self.failmsg("test2v1", "bad bulkdata table")
  142. self.failmsg("test2v2", "empty, with corrupted format file", fix=False)
  143. self.okmsg("test2v2", "empty, with corrupted format file")
  144. self.failmsg("test2w1", "out of range, and zero", fix=False)
  145. self.okmsg("test2w1", "Will try truncating table")
  146. self.contain("Deleting the entire interval")
  147. self.failmsg("test2w2", "non-monotonic, and zero", fix=False)
  148. self.okmsg("test2w2", "Will try truncating table")
  149. self.contain("new end: time 237000001, pos 238")