Add fsck scan for any data timestamps outside interval range

nilmdb/fsck/fsck.py

@@ -468,23 +468,35 @@ class Fsck(object):
                         "%s: failed to grab rows %d through %d: %s",
                         path, start, stop, repr(e))
 
+                ts = data['timestamp']
+
+                # Verify that all timestamps are in range.
+                match = (ts < stime) | (ts >= etime)
+                if match.any():
+                    row = start + numpy.argmax(match)
+                    raise FsckError("%s: data timestamp %d at row %d "
+                                    "outside interval range [%d,%d)",
+                                    path, data['timestamp'][row], row,
+                                    stime, etime)
+
                 # Verify that timestamps are monotonic
-                if (numpy.diff(data['timestamp']) <= 0).any():
-                    raise FsckError("%s: non-monotonic timestamp(s) in rows "
-                                    "%d through %d", path, start, stop)
-                first_ts = data['timestamp'][0]
-                print("first_ts", first_ts, "last_ts", last_ts)
+                match = numpy.diff(ts) <= 0
+                if match.any():
+                    row = start + numpy.argmax(match)
+                    raise FsckError("%s: non-monotonic timestamp (%d -> %d) "
+                                    "at row %d", path, ts[row], ts[row+1], row)
+                first_ts = ts[0]
                 if last_ts is not None and first_ts <= last_ts:
                     raise FsckError("%s: first interval timestamp %d is not "
                                     "greater than the previous last interval "
                                     "timestamp %d, at row %d",
                                     path, first_ts, last_ts, start)
-                last_ts = data['timestamp'][-1]
-                print("last_ts", last_ts)
+                last_ts = ts[-1]
 
-                # These are probably fixable, by removing the offending
-                # intervals.  But I'm not going to bother implementing
-                # that yet.
+                # The previous errors are fixable, by removing the
+                # offending intervals, or changing the data
+                # timestamps.  But these are probably unlikely errors,
+                # so it's not worth implementing that yet.
 
                 # Done
                 done += count

tests/test_fsck.py

@@ -143,7 +143,7 @@ class TestFsck(object):
         self.contain("restarting fsck")
         self.ok("test2r2")
 
-        self.failmsg("test2s", "non-monotonic timestamp(s)")
+        self.failmsg("test2s", "non-monotonic timestamp (1000000 -> 12345)")
 
         def check_small_maxrows(f):
             f.maxrows_override = 1
@@ -153,6 +153,9 @@ class TestFsck(object):
 
         self.ok("test2t", skip=True)
 
+        self.failmsg("test2u", "data timestamp 1234567890 at row 28 outside")
+        self.failmsg("test2u1", "data timestamp 7 at row 0 outside")
+
         @nilmdb.fsck.fsck.retry_if_raised(Exception, max_retries=3)
         def foo():
             raise Exception("hi")