Add nilm-cleanup test, update for Python 3

Also fix a bug when reporting bad units on "keep" configuration key

nilmtools/cleanup.py

@@ -14,6 +14,7 @@ import sys
 import collections
 import fnmatch
 import re
+import os
 
 def warn(msg, *args):
     fprintf(sys.stderr, "warning: " + msg + "\n", *args)
@@ -33,7 +34,7 @@ class TimePeriod(object):
                 self.count = float(val[:-len(u)])
                 break
         else:
-            raise ValueError("unknown units: " + units)
+            raise ValueError("unknown units: " + val)
 
     def seconds(self):
         return self.count * self.scale
@@ -65,7 +66,6 @@ class StreamCleanupConfig(object):
 def main(argv = None):
     parser = argparse.ArgumentParser(
         formatter_class = argparse.RawDescriptionHelpFormatter,
-        version = nilmtools.__version__,
         description = """\
     Clean up old data from streams using a configuration file to specify
     which data to remove.
@@ -93,8 +93,10 @@ def main(argv = None):
 
     Rate is optional and is only used for the --estimate option.
     """)
-    parser.add_argument("-u", "--url", action="store",
-                        default="http://localhost/nilmdb/",
+    parser.add_argument("-v", "--version", action="version",
+                        version=nilmtools.__version__)
+    def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
+    parser.add_argument("-u", "--url", action="store", default=def_url,
                         help="NilmDB server URL (default: %(default)s)")
     parser.add_argument("-y", "--yes", action="store_true",
                         default = False,

tests/data/cleanup-bad.cfg

@@ -0,0 +1,2 @@
+[/newton/*]
+keep = 3

tests/data/cleanup-nodecim.cfg

@@ -0,0 +1,4 @@
+[/newton/*]
+keep = 3w
+rate = 8000
+decimated = false

tests/data/cleanup.cfg

@@ -0,0 +1,13 @@
+[/newton/*]
+keep = 3w
+rate = 8000
+decimated = true
+
+[/sf/*]
+keep = 0.01h
+dummy = xxx
+
+[/empty/foo]
+
+[/nonexistent/bar]
+

tests/test.py

@@ -245,7 +245,7 @@ class TestAllCommands(CommandTester):
         eq_(len(list(client2.stream_extract("/newton/prep"))), 0)
 
         # this should copy a bunch
-        self.ok(f"-u {self.url} -U {self.url2} /newton*")
+        self.ok(f"-u {self.url} -U {self.url2} /*")
         self.contain("Creating destination stream /newton/prep-copy", False)
         eq_(client1.stream_list(), client2.stream_list())
         eq_(list(client1.stream_extract("/newton/prep")),
@@ -553,6 +553,33 @@ class TestAllCommands(CommandTester):
 
     def test_08_cleanup(self):
         self.main = nilmtools.cleanup.main
+        client = nilmdb.client.Client(url=self.url)
+
+        # This mostly just gets coverage, doesn't carefully verify behavior
+        self.fail(f"")
+        self.ok(f"--help")
+
+        self.fail(f"tests/data/cleanup-bad.cfg")
+        self.contain("unknown units")
+
+        client.stream_create("/empty/foo", "uint16_1")
+        self.ok(f"tests/data/cleanup.cfg")
+        self.contain("'/nonexistent/bar' did not match any existing streams")
+        self.contain("no config for existing stream '/empty/foo'")
+        self.contain("nothing to do (only 0.00 weeks of data present)")
+        self.contain("specify --yes to actually perform")
+
+        self.ok(f"--yes tests/data/cleanup.cfg")
+        self.contain("removing data before")
+        self.contain("removing from /sf/raw")
+
+        self.ok(f"--estimate tests/data/cleanup.cfg")
+        self.contain("Total estimated disk usage")
+        self.contain("MiB")
+        self.contain("GiB")
+
+        self.ok(f"--yes tests/data/cleanup-nodecim.cfg")
+        self.ok(f"--estimate tests/data/cleanup-nodecim.cfg")
 
     def test_09_trainola(self):
         self.main = nilmtools.trainola.main