Prevent negative numbers in dbinfo output.

This might occur if things change while we're calculating the sizes.

Makefile

@@ -1,5 +1,5 @@
 # By default, run the tests.
-all: fscktest
+all: test
 
 version:
 	python setup.py version
@@ -23,10 +23,6 @@ docs:
 lint:
 	pylint --rcfile=.pylintrc nilmdb
 
-fscktest:
-	python -c "import nilmdb.fsck; nilmdb.fsck.Fsck('/home/jim/wsgi/db').check()"
-#	python -c "import nilmdb.fsck; nilmdb.fsck.Fsck('/home/jim/mnt/bucket/mnt/sharon/data/db', True).check()"
-
 test:
 ifeq ($(INSIDE_EMACS), t)
 # Use the slightly more flexible script

nilmdb/cmdline/cmdline.py

@@ -19,9 +19,8 @@ except ImportError: # pragma: no cover
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "help", "info", "create", "list", "metadata",
+subcommands = [ "help", "info", "create", "rename", "list", "intervals",
-                "insert", "extract", "remove", "destroy",
+                "metadata", "insert", "extract", "remove", "destroy" ]
-                "intervals", "rename" ]
 
 # Import the subcommand modules
 subcmd_mods = {}

nilmdb/fsck/fsck.py

@@ -44,15 +44,16 @@ def err(format, *args):
     fprintf(sys.stderr, format, *args)
 
 # Decorator that retries a function if it returns a specific value
-def retry_if_raised(exc, message = None):
+def retry_if_raised(exc, message = None, max_retries = 100):
     def f1(func):
         def f2(*args, **kwargs):
-            while True:
+            for n in range(max_retries):
                 try:
                     return func(*args, **kwargs)
                 except exc as e:
                     if message:
                         log("%s\n\n", message)
+            raise Exception("Max number of retries (%d) exceeded; giving up")
         return f2
     return f1
 
@@ -89,7 +90,7 @@ class Fsck(object):
     ### Main checks
 
     @retry_if_raised(RetryFsck, "Something was fixed: restarting fsck")
-    def check(self):
+    def check(self, skip_data = False):
         self.bulk = None
         self.sql = None
         try:
@@ -97,6 +98,9 @@ class Fsck(object):
             self.check_sql()
             self.check_streams()
             self.check_intervals()
+            if skip_data:
+                log("skipped data check\n")
+            else:
                 self.check_data()
         finally:
             if self.bulk:
@@ -118,7 +122,11 @@ class Fsck(object):
             raise FsckError("Bulk data directory missing (%s)", self.bulkpath)
         with open(self.bulklock, "w") as lockfile:
             if not nilmdb.utils.lock.exclusive_lock(lockfile):
-                raise FsckError('database already locked by another process')
+                raise FsckError('Database already locked by another process\n'
+                                'Make sure all other processes that might be '
+                                'using the database are stopped.\n'
+                                'Restarting apache will cause it to unlock '
+                                'the db until a request is received.')
             # unlocked immediately
         self.bulk = nilmdb.server.bulkdata.BulkData(self.basepath)
 
@@ -170,7 +178,7 @@ class Fsck(object):
 
     def check_streams(self):
         ids = self.stream_path.keys()
-        log("checking %d streams\n", len(ids))
+        log("checking %s streams\n", "{:,d}".format(len(ids)))
         with Progress(len(ids)) as pbar:
             for i, sid in enumerate(ids):
                 pbar.update(i)
@@ -306,7 +314,7 @@ class Fsck(object):
 
     def check_intervals(self):
         total_ints = sum(len(x) for x in self.stream_interval.values())
-        log("checking %d intervals\n", total_ints)
+        log("checking %s intervals\n", "{:,d}".format(total_ints))
         done = 0
         with Progress(total_ints) as pbar:
             for sid in self.stream_interval:
@@ -389,7 +397,7 @@ class Fsck(object):
     def check_data(self):
         total_rows = sum(sum((y[3] - y[2]) for y in x)
                          for x in self.stream_interval.values())
-        log("checking %d rows of data\n", total_rows)
+        log("checking %s rows of data\n", "{:,d}".format(total_rows))
         done = 0
         with Progress(total_rows) as pbar:
             for sid in self.stream_interval:

nilmdb/scripts/nilmdb_fsck.py

@@ -16,10 +16,12 @@ def main():
     parser.add_argument("-f", "--fix", action="store_true",
                         default=False, help = 'Fix errors when possible '
                         '(which may involve removing data)')
+    parser.add_argument("-n", "--no-data", action="store_true",
+                        default=False, help = 'Skip the slow full-data check')
     parser.add_argument('database', help = 'Database directory')
     args = parser.parse_args()
 
-    nilmdb.fsck.Fsck(args.database, args.fix).check()
+    nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data = args.no_data)
 
 if __name__ == "__main__":
     main()

nilmdb/server/server.py

@@ -74,8 +74,8 @@ class Root(NilmApp):
         dbsize = nilmdb.utils.du(path)
         return { "path": path,
                  "size": dbsize,
-                 "other": usage.used - dbsize,
+                 "other": max(usage.used - dbsize, 0),
-                 "reserved": usage.total - usage.used - usage.free,
+                 "reserved": max(usage.total - usage.used - usage.free, 0),
                  "free": usage.free }
 
 class Stream(NilmApp):

nilmdb/utils/diskusage.py

@@ -21,7 +21,8 @@ def du(path):
     errors that might occur if we encounter broken symlinks or
     files in the process of being removed."""
     try:
-        size = os.path.getsize(path)
+        st = os.stat(path)
+        size = st.st_blocks * 512
         if os.path.isdir(path):
             for thisfile in os.listdir(path):
                 filepath = os.path.join(path, thisfile)