fsck: limit max retries so we don't get stuck in a loop forever

This makes it fast enough to run at startup with -f, if it's expected
that a system will frequently need to be fixed.

nilmdb/fsck/fsck.py

@@ -44,15 +44,16 @@ def err(format, *args):
     fprintf(sys.stderr, format, *args)
 
 # Decorator that retries a function if it returns a specific value
-def retry_if_raised(exc, message = None):
+def retry_if_raised(exc, message = None, max_retries = 100):
     def f1(func):
         def f2(*args, **kwargs):
-            while True:
+            for n in range(max_retries):
                 try:
                     return func(*args, **kwargs)
                 except exc as e:
                     if message:
                         log("%s\n\n", message)
+            raise Exception("Max number of retries (%d) exceeded; giving up")
         return f2
     return f1
 
@@ -89,7 +90,7 @@ class Fsck(object):
     ### Main checks
 
     @retry_if_raised(RetryFsck, "Something was fixed: restarting fsck")
-    def check(self):
+    def check(self, skip_data = False):
         self.bulk = None
         self.sql = None
         try:
@@ -97,6 +98,9 @@ class Fsck(object):
             self.check_sql()
             self.check_streams()
             self.check_intervals()
+            if skip_data:
+                log("skipped data check\n")
+            else:
                 self.check_data()
         finally:
             if self.bulk:
@@ -118,7 +122,11 @@ class Fsck(object):
             raise FsckError("Bulk data directory missing (%s)", self.bulkpath)
         with open(self.bulklock, "w") as lockfile:
             if not nilmdb.utils.lock.exclusive_lock(lockfile):
-                raise FsckError('database already locked by another process')
+                raise FsckError('Database already locked by another process\n'
+                                'Make sure all other processes that might be '
+                                'using the database are stopped.\n'
+                                'Restarting apache will cause it to unlock '
+                                'the db until a request is received.')
             # unlocked immediately
         self.bulk = nilmdb.server.bulkdata.BulkData(self.basepath)
 
@@ -170,7 +178,7 @@ class Fsck(object):
 
     def check_streams(self):
         ids = self.stream_path.keys()
-        log("checking %d streams\n", len(ids))
+        log("checking %s streams\n", "{:,d}".format(len(ids)))
         with Progress(len(ids)) as pbar:
             for i, sid in enumerate(ids):
                 pbar.update(i)
@@ -306,7 +314,7 @@ class Fsck(object):
 
     def check_intervals(self):
         total_ints = sum(len(x) for x in self.stream_interval.values())
-        log("checking %d intervals\n", total_ints)
+        log("checking %s intervals\n", "{:,d}".format(total_ints))
         done = 0
         with Progress(total_ints) as pbar:
             for sid in self.stream_interval:
@@ -389,7 +397,7 @@ class Fsck(object):
     def check_data(self):
         total_rows = sum(sum((y[3] - y[2]) for y in x)
                          for x in self.stream_interval.values())
-        log("checking %d rows of data\n", total_rows)
+        log("checking %s rows of data\n", "{:,d}".format(total_rows))
         done = 0
         with Progress(total_rows) as pbar:
             for sid in self.stream_interval:

nilmdb/scripts/nilmdb_fsck.py

@@ -16,10 +16,12 @@ def main():
     parser.add_argument("-f", "--fix", action="store_true",
                         default=False, help = 'Fix errors when possible '
                         '(which may involve removing data)')
+    parser.add_argument("-n", "--no-data", action="store_true",
+                        default=False, help = 'Skip the slow full-data check')
     parser.add_argument('database', help = 'Database directory')
     args = parser.parse_args()
 
-    nilmdb.fsck.Fsck(args.database, args.fix).check()
+    nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data = args.no_data)
 
 if __name__ == "__main__":
     main()