Test --no-decim

We saw a bug where renamed streams had missing data at the end.  I
think what happened is:

- Write data to /old/path
- Rename to /new/path
- Write data to /new/path
- Cache entry for /old/path gets evicted, file gets truncated

Instead, make sure we evict /old/path right away when renaming.

Makefile

@@ -1,5 +1,5 @@
 # By default, run the tests.
-all: fscktest
+all: test
 
 version:
 	python setup.py version
@@ -23,10 +23,6 @@ docs:
 lint:
 	pylint --rcfile=.pylintrc nilmdb
 
-fscktest:
-	python -c "import nilmdb.fsck; nilmdb.fsck.Fsck('/home/jim/wsgi/db').check()"
-#	python -c "import nilmdb.fsck; nilmdb.fsck.Fsck('/home/jim/mnt/bucket/mnt/sharon/data/db', True).check()"
-
 test:
 ifeq ($(INSIDE_EMACS), t)
 # Use the slightly more flexible script

nilmdb/client/httpclient.py

@@ -9,7 +9,7 @@ import requests
 
 class HTTPClient(object):
     """Class to manage and perform HTTP requests from the client"""
-    def __init__(self, baseurl = "", post_json = False):
+    def __init__(self, baseurl = "", post_json = False, verify_ssl = True):
         """If baseurl is supplied, all other functions that take
         a URL can be given a relative URL instead."""
         # Verify / clean up URL
@@ -19,6 +19,7 @@ class HTTPClient(object):
         self.baseurl = reparsed.rstrip('/') + '/'
 
         # Build Requests session object, enable SSL verification
+        self.verify_ssl = verify_ssl
         self.session = requests.Session()
         self.session.verify = True
 
@@ -67,7 +68,8 @@ class HTTPClient(object):
                                             params = query_data,
                                             data = body_data,
                                             stream = stream,
-                                            headers = headers)
+                                            headers = headers,
+                                            verify = self.verify_ssl)
         except requests.RequestException as e:
             raise ServerError(status = "502 Error", url = url,
                               message = str(e.message))

nilmdb/cmdline/cmdline.py

@@ -19,9 +19,8 @@ except ImportError: # pragma: no cover
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "help", "info", "create", "list", "metadata",
-                "insert", "extract", "remove", "destroy",
-                "intervals", "rename" ]
+subcommands = [ "help", "info", "create", "rename", "list", "intervals",
+                "metadata", "insert", "extract", "remove", "destroy" ]
 
 # Import the subcommand modules
 subcmd_mods = {}
@@ -122,7 +121,7 @@ class Cmdline(object):
         group = self.parser.add_argument_group("General options")
         group.add_argument("-h", "--help", action='help',
                            help='show this help message and exit')
-        group.add_argument("-V", "--version", action="version",
+        group.add_argument("-v", "--version", action="version",
                            version = nilmdb.__version__)
 
         group = self.parser.add_argument_group("Server")

nilmdb/cmdline/list.py

@@ -45,6 +45,8 @@ def setup(self, sub):
                        help="Show raw timestamps when printing times")
     group.add_argument("-l", "--layout", action="store_true",
                        help="Show layout type next to path name")
+    group.add_argument("-n", "--no-decim", action="store_true",
+                       help="Skip paths containing \"~decim-\"")
 
     return cmd
 
@@ -71,6 +73,8 @@ def cmd_list(self):
             (path, layout, int_min, int_max, rows, time) = stream[:6]
             if not fnmatch.fnmatch(path, argpath):
                 continue
+            if self.args.no_decim and "~decim-" in path:
+                continue
 
             if self.args.layout:
                 printf("%s %s\n", path, layout)

nilmdb/fsck/fsck.py

@@ -44,15 +44,16 @@ def err(format, *args):
     fprintf(sys.stderr, format, *args)
 
 # Decorator that retries a function if it returns a specific value
-def retry_if_raised(exc, message = None):
+def retry_if_raised(exc, message = None, max_retries = 100):
     def f1(func):
         def f2(*args, **kwargs):
-            while True:
+            for n in range(max_retries):
                 try:
                     return func(*args, **kwargs)
                 except exc as e:
                     if message:
                         log("%s\n\n", message)
+            raise Exception("Max number of retries (%d) exceeded; giving up")
         return f2
     return f1
 
@@ -89,7 +90,7 @@ class Fsck(object):
     ### Main checks
 
     @retry_if_raised(RetryFsck, "Something was fixed: restarting fsck")
-    def check(self):
+    def check(self, skip_data = False):
         self.bulk = None
         self.sql = None
         try:
@@ -97,7 +98,10 @@ class Fsck(object):
             self.check_sql()
             self.check_streams()
             self.check_intervals()
-            self.check_data()
+            if skip_data:
+                log("skipped data check\n")
+            else:
+                self.check_data()
         finally:
             if self.bulk:
                 self.bulk.close()
@@ -118,7 +122,11 @@ class Fsck(object):
             raise FsckError("Bulk data directory missing (%s)", self.bulkpath)
         with open(self.bulklock, "w") as lockfile:
             if not nilmdb.utils.lock.exclusive_lock(lockfile):
-                raise FsckError('database already locked by another process')
+                raise FsckError('Database already locked by another process\n'
+                                'Make sure all other processes that might be '
+                                'using the database are stopped.\n'
+                                'Restarting apache will cause it to unlock '
+                                'the db until a request is received.')
             # unlocked immediately
         self.bulk = nilmdb.server.bulkdata.BulkData(self.basepath)
 
@@ -170,7 +178,7 @@ class Fsck(object):
 
     def check_streams(self):
         ids = self.stream_path.keys()
-        log("checking %d streams\n", len(ids))
+        log("checking %s streams\n", "{:,d}".format(len(ids)))
         with Progress(len(ids)) as pbar:
             for i, sid in enumerate(ids):
                 pbar.update(i)
@@ -306,7 +314,7 @@ class Fsck(object):
 
     def check_intervals(self):
         total_ints = sum(len(x) for x in self.stream_interval.values())
-        log("checking %d intervals\n", total_ints)
+        log("checking %s intervals\n", "{:,d}".format(total_ints))
         done = 0
         with Progress(total_ints) as pbar:
             for sid in self.stream_interval:
@@ -373,7 +381,7 @@ class Fsck(object):
         err("*** Deleting the entire interval from SQL.\n")
         err("This may leave stale data on disk.  To fix that, copy all\n")
         err("data from this stream to a new stream, then remove all data\n")
-        err("from and destroy %s.\n")
+        err("from and destroy %s.\n", path)
         with self.sql:
             cur = self.sql.cursor()
             cur.execute("DELETE FROM ranges WHERE "
@@ -389,7 +397,7 @@ class Fsck(object):
     def check_data(self):
         total_rows = sum(sum((y[3] - y[2]) for y in x)
                          for x in self.stream_interval.values())
-        log("checking %d rows of data\n", total_rows)
+        log("checking %s rows of data\n", "{:,d}".format(total_rows))
         done = 0
         with Progress(total_rows) as pbar:
             for sid in self.stream_interval:

nilmdb/scripts/nilmdb_fsck.py

@@ -10,16 +10,17 @@ def main():
 
     parser = argparse.ArgumentParser(
         description = 'Check database consistency',
-        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument("-V", "--version", action="version",
-                        version = nilmdb.__version__)
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter,
+        version = nilmdb.__version__)
     parser.add_argument("-f", "--fix", action="store_true",
                         default=False, help = 'Fix errors when possible '
                         '(which may involve removing data)')
+    parser.add_argument("-n", "--no-data", action="store_true",
+                        default=False, help = 'Skip the slow full-data check')
     parser.add_argument('database', help = 'Database directory')
     args = parser.parse_args()
 
-    nilmdb.fsck.Fsck(args.database, args.fix).check()
+    nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data = args.no_data)
 
 if __name__ == "__main__":
     main()

nilmdb/scripts/nilmdb_server.py

@@ -10,10 +10,8 @@ def main():
 
     parser = argparse.ArgumentParser(
         description = 'Run the NilmDB server',
-        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
-
-    parser.add_argument("-V", "--version", action="version",
-                        version = nilmdb.__version__)
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter,
+        version = nilmdb.__version__)
 
     group = parser.add_argument_group("Standard options")
     group.add_argument('-a', '--address',

nilmdb/server/bulkdata.py

@@ -194,6 +194,9 @@ class BulkData(object):
         if oldospath == newospath:
             raise ValueError("old and new paths are the same")
 
+        # Remove Table object at old path from cache
+        self.getnode.cache_remove(self, oldunicodepath)
+
         # Move the table to a temporary location
         tmpdir = tempfile.mkdtemp(prefix = "rename-", dir = self.root)
         tmppath = os.path.join(tmpdir, "table")

nilmdb/server/server.py

@@ -74,8 +74,8 @@ class Root(NilmApp):
         dbsize = nilmdb.utils.du(path)
         return { "path": path,
                  "size": dbsize,
-                 "other": usage.used - dbsize,
-                 "reserved": usage.total - usage.used - usage.free,
+                 "other": max(usage.used - dbsize, 0),
+                 "reserved": max(usage.total - usage.used - usage.free, 0),
                  "free": usage.free }
 
 class Stream(NilmApp):

nilmdb/utils/diskusage.py

@@ -21,7 +21,8 @@ def du(path):
     errors that might occur if we encounter broken symlinks or
     files in the process of being removed."""
     try:
-        size = os.path.getsize(path)
+        st = os.stat(path)
+        size = st.st_blocks * 512
         if os.path.isdir(path):
             for thisfile in os.listdir(path):
                 filepath = os.path.join(path, thisfile)

tests/test_cmdline.py

@@ -290,6 +290,7 @@ class TestCmdline(object):
         self.ok("create /newton/zzz/rawnotch uint16_9")
         self.ok("create /newton/prep float32_8")
         self.ok("create /newton/raw uint16_6")
+        self.ok("create /newton/raw~decim-1234 uint16_6")
 
         # Create a stream that already exists
         self.fail("create /newton/raw uint16_6")
@@ -305,13 +306,23 @@ class TestCmdline(object):
         self.fail("create /newton/zzz float32_8")
         self.contain("subdirs of this path already exist")
 
-        # Verify we got those 3 streams and they're returned in
+        # Verify we got those 4 streams and they're returned in
         # alphabetical order.
         self.ok("list -l")
         self.match("/newton/prep float32_8\n"
                    "/newton/raw uint16_6\n"
+                   "/newton/raw~decim-1234 uint16_6\n"
                    "/newton/zzz/rawnotch uint16_9\n")
 
+        # No decimated streams if -n specified
+        self.ok("list -n -l")
+        self.match("/newton/prep float32_8\n"
+                   "/newton/raw uint16_6\n"
+                   "/newton/zzz/rawnotch uint16_9\n")
+
+        # Delete that decimated stream
+        self.ok("destroy /newton/raw~decim-1234")
+
         # Match just one type or one path.  Also check
         # that --path is optional
         self.ok("list --layout /newton/raw")