Fix streaming requests like stream_extract

This is so tests can be run without interfering with a normal server.

nilmdb/client/client.py

@@ -52,12 +52,14 @@ class Client(object):
         as a dictionary."""
         return self.http.get("dbinfo")
 
-    def stream_list(self, path = None, layout = None):
+    def stream_list(self, path = None, layout = None, extent = False):
         params = {}
         if path is not None:
             params["path"] = path
         if layout is not None:
             params["layout"] = layout
+        if extent:
+            params["extent"] = 1
         return self.http.get("stream/list", params)
 
     def stream_get_metadata(self, path, keys = None):

nilmdb/client/httpclient.py

@@ -62,7 +62,8 @@ class HTTPClient(object):
         try:
             response = self.session.request(method, url,
                                             params = query_data,
-                                            data = body_data)
+                                            data = body_data,
+                                            stream = stream)
         except requests.RequestException as e:
             raise ServerError(status = "502 Error", url = url,
                               message = str(e.message))

nilmdb/cmdline/cmdline.py

@@ -6,13 +6,14 @@ from nilmdb.utils import datetime_tz
 import nilmdb.utils.time
 
 import sys
+import os
 import argparse
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
+subcommands = [ "help", "info", "create", "list", "metadata",
-                "remove", "destroy" ]
+                "insert", "extract", "remove", "destroy" ]
 
 # Import the subcommand modules
 subcmd_mods = {}
@@ -29,6 +30,8 @@ class Cmdline(object):
     def __init__(self, argv = None):
         self.argv = argv or sys.argv[1:]
         self.client = None
+        self.def_url = os.environ.get("NILMDB_URL", "http://localhost:12380")
+        self.subcmd = {}
 
     def arg_time(self, toparse):
         """Parse a time string argument"""
@@ -50,18 +53,17 @@ class Cmdline(object):
 
         group = self.parser.add_argument_group("Server")
         group.add_argument("-u", "--url", action="store",
-                           default="http://localhost:12380/",
+                           default=self.def_url,
                            help="NilmDB server URL (default: %(default)s)")
 
-        sub = self.parser.add_subparsers(title="Commands",
+        sub = self.parser.add_subparsers(
-                                         dest="command",
+            title="Commands", dest="command",
-                                         description="Specify --help after "
+            description="Use 'help command' or 'command --help' for more "
-                                         "the command for command-specific "
+            "details on a particular command.")
-                                         "options.")
 
         # Set up subcommands (defined in separate files)
         for cmd in subcommands:
-            subcmd_mods[cmd].setup(self, sub)
+            self.subcmd[cmd] = subcmd_mods[cmd].setup(self, sub)
 
     def die(self, formatstr, *args):
         fprintf(sys.stderr, formatstr + "\n", *args)
@@ -84,11 +86,13 @@ class Cmdline(object):
 
         self.client = nilmdb.Client(self.args.url)
 
-        # Make a test connection to make sure things work
+        # Make a test connection to make sure things work,
-        try:
+        # unless the particular command requests that we don't.
-            server_version = self.client.version()
+        if "no_test_connect" not in self.args:
-        except nilmdb.client.Error as e:
+            try:
-            self.die("error connecting to server: %s", str(e))
+                server_version = self.client.version()
+            except nilmdb.client.Error as e:
+                self.die("error connecting to server: %s", str(e))
 
         # Now dispatch client request to appropriate function.  Parser
         # should have ensured that we don't have any unknown commands

nilmdb/cmdline/create.py

@@ -26,6 +26,7 @@ Layout types are of the format: type_count
                        help="Path (in database) of new stream, e.g. /foo/bar")
     group.add_argument("layout",
                        help="Layout type for new stream, e.g. float32_8")
+    return cmd
 
 def cmd_create(self):
     """Create new stream"""

nilmdb/cmdline/destroy.py

@@ -16,6 +16,7 @@ def setup(self, sub):
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
                        help="Path of the stream to delete, e.g. /foo/bar")
+    return cmd
 
 def cmd_destroy(self):
     """Destroy stream"""

nilmdb/cmdline/extract.py

@@ -30,6 +30,7 @@ def setup(self, sub):
                        help="Show raw timestamps in annotated information")
     group.add_argument("-c", "--count", action="store_true",
                        help="Just output a count of matched data points")
+    return cmd
 
 def cmd_extract_verify(self):
     if self.args.start is not None and self.args.end is not None:

nilmdb/cmdline/help.py

@@ -0,0 +1,26 @@
+from nilmdb.utils.printf import *
+
+import argparse
+import sys
+
+def setup(self, sub):
+    cmd = sub.add_parser("help", help="Show detailed help for a command",
+                         description="""
+                         Show help for a command. 'help command' is
+                         the same as 'command --help'.
+                         """)
+    cmd.set_defaults(handler = cmd_help)
+    cmd.set_defaults(no_test_connect = True)
+    cmd.add_argument("command", nargs="?",
+                     help="Command to get help about")
+    cmd.add_argument("rest", nargs=argparse.REMAINDER,
+                     help=argparse.SUPPRESS)
+    return cmd
+
+def cmd_help(self):
+    if self.args.command in self.subcmd:
+        self.subcmd[self.args.command].print_help()
+    else:
+        self.parser.print_help()
+
+    return

nilmdb/cmdline/info.py

@@ -12,6 +12,7 @@ def setup(self, sub):
                          version.
                          """)
     cmd.set_defaults(handler = cmd_info)
+    return cmd
 
 def cmd_info(self):
     """Print info about the server"""

nilmdb/cmdline/insert.py

@@ -47,6 +47,7 @@ def setup(self, sub):
                        help="Path of stream, e.g. /foo/bar")
     group.add_argument("file", nargs="*", default=['-'],
                        help="File(s) to insert (default: - (stdin))")
+    return cmd
 
 def cmd_insert(self):
     # Find requested stream

nilmdb/cmdline/list.py

@@ -24,11 +24,13 @@ def setup(self, sub):
     group.add_argument("-l", "--layout", default="*",
                        help="Match only this stream layout")
 
+    group = cmd.add_argument_group("Interval extent")
+    group.add_argument("-E", "--extent", action="store_true",
+                       help="Show min/max timestamps in this stream")
+
     group = cmd.add_argument_group("Interval details")
     group.add_argument("-d", "--detail", action="store_true",
                        help="Show available data time intervals")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps in time intervals")
     group.add_argument("-s", "--start",
                        metavar="TIME", type=self.arg_time,
                        help="Starting timestamp (free-form, inclusive)")
@@ -36,6 +38,12 @@ def setup(self, sub):
                        metavar="TIME", type=self.arg_time,
                        help="Ending timestamp (free-form, noninclusive)")
 
+    group = cmd.add_argument_group("Misc options")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps in time intervals or extents")
+
+    return cmd
+
 def cmd_list_verify(self):
     # A hidden "path_positional" argument lets the user leave off the
     # "-p" when specifying the path.  Handle it here.
@@ -51,28 +59,38 @@ def cmd_list_verify(self):
         if self.args.start >= self.args.end:
             self.parser.error("start must precede end")
 
+    if self.args.start is not None or self.args.end is not None:
+        if not self.args.detail:
+            self.parser.error("--start and --end only make sense with --detail")
+
 def cmd_list(self):
     """List available streams"""
-    streams = self.client.stream_list()
+    streams = self.client.stream_list(extent = True)
 
     if self.args.timestamp_raw:
         time_string = repr
     else:
         time_string = nilmdb.utils.time.format_time
 
-    for (path, layout) in streams:
+    for (path, layout, extent_min, extent_max) in streams:
         if not (fnmatch.fnmatch(path, self.args.path) and
                 fnmatch.fnmatch(layout, self.args.layout)):
             continue
 
         printf("%s %s\n", path, layout)
-        if not self.args.detail:
-            continue
 
-        printed = False
+        if self.args.extent:
-        for (start, end) in self.client.stream_intervals(path, self.args.start,
+            if extent_min is None or extent_max is None:
-                                                         self.args.end):
+                printf("  extent: (no data)\n")
-            printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
+            else:
-            printed = True
+                printf("  extent: %s -> %s\n",
-        if not printed:
+                       time_string(extent_min), time_string(extent_max))
-            printf("  (no intervals)\n")
+
+        if self.args.detail:
+            printed = False
+            for (start, end) in self.client.stream_intervals(
+                path, self.args.start, self.args.end):
+                printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
+                printed = True
+            if not printed:
+                printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -26,6 +26,7 @@ def setup(self, sub):
     exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
                      help="Update metadata using provided "
                      "key=value pairs")
+    return cmd
 
 def cmd_metadata(self):
     """Manipulate metadata"""

nilmdb/cmdline/remove.py

@@ -23,6 +23,7 @@ def setup(self, sub):
     group = cmd.add_argument_group("Output format")
     group.add_argument("-c", "--count", action="store_true",
                        help="Output number of data points removed")
+    return cmd
 
 def cmd_remove(self):
     try:

nilmdb/server/nilmdb.py

@@ -269,28 +269,39 @@ class NilmDB(object):
 
         return
 
-    def stream_list(self, path = None, layout = None):
+    def stream_list(self, path = None, layout = None, extent = False):
-        """Return list of [path, layout] lists of all streams
+        """Return list of lists of all streams in the database.
-        in the database.
 
         If path is specified, include only streams with a path that
         matches the given string.
 
         If layout is specified, include only streams with a layout
         that matches the given string.
-        """
-        where = "WHERE 1=1"
-        params = ()
-        if layout:
-            where += " AND layout=?"
-            params += (layout,)
-        if path:
-            where += " AND path=?"
-            params += (path,)
-        result = self.con.execute("SELECT path, layout "
-                                  "FROM streams " + where, params).fetchall()
 
-        return sorted(list(x) for x in result)
+        If extent = False, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extent = True, returns a list of lists containing the
+        path, layout, and min/max extent of the data:
+        [ path, layout, extent_min, extent_max ]
+        """
+        params = ()
+        query = "SELECT streams.path, streams.layout"
+        if extent:
+            query += ", min(ranges.start_time), max(ranges.end_time)"
+        query += " FROM streams"
+        if extent:
+            query += " LEFT JOIN ranges ON streams.id = ranges.stream_id"
+        query += " WHERE 1=1"
+        if layout is not None:
+            query += " AND streams.layout=?"
+            params += (layout,)
+        if path is not None:
+            query += " AND streams.path=?"
+            params += (path,)
+        query += " GROUP BY streams.id ORDER BY streams.path"
+        result = self.con.execute(query, params).fetchall()
+        return [ list(x) for x in result ]
 
     def stream_intervals(self, path, start = None, end = None):
         """

nilmdb/server/server.py

@@ -121,14 +121,22 @@ class Stream(NilmApp):
 
     # /stream/list
     # /stream/list?layout=PrepData
-    # /stream/list?path=/newton/prep
+    # /stream/list?path=/newton/prep&extent=1
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def list(self, path = None, layout = None):
+    def list(self, path = None, layout = None, extent = None):
         """List all streams in the database.  With optional path or
         layout parameter, just list streams that match the given path
-        or layout"""
+        or layout.
-        return self.db.stream_list(path, layout)
+
+        If extent is not given, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extent is provided, returns a list of lists containing the
+        path, layout, and min/max extent of the data:
+        [ path, layout, extent_min, extent_max ]
+        """
+        return self.db.stream_list(path, layout, bool(extent))
 
     # /stream/create?path=/newton/prep&layout=PrepData
     @cherrypy.expose

tests/test_client.py

@@ -25,7 +25,7 @@ import re
 from testutil.helpers import *
 
 testdb = "tests/client-testdb"
-testurl = "http://localhost:12380/"
+testurl = "http://localhost:32180/"
 
 def setup_module():
     global test_server, test_db
@@ -35,7 +35,7 @@ def setup_module():
     # Start web app on a custom port
     test_db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(testdb, sync = False)
     test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+                                port = 32180, stoppable = False,
                                 fast_shutdown = True,
                                 force_traceback = False)
     test_server.start(blocking = False)
@@ -62,13 +62,13 @@ class TestClient(object):
         client.close()
 
         # Then a fake URL on a real host
-        client = nilmdb.Client(url = "http://localhost:12380/fake/")
+        client = nilmdb.Client(url = "http://localhost:32180/fake/")
         with assert_raises(nilmdb.client.ClientError):
             client.version()
         client.close()
 
         # Now a real URL with no http:// prefix
-        client = nilmdb.Client(url = "localhost:12380")
+        client = nilmdb.Client(url = "localhost:32180")
         version = client.version()
         client.close()
 
@@ -584,7 +584,7 @@ class TestClient(object):
             def connections():
                 try:
                     poolmanager = c.http._last_response.connection.poolmanager
-                    pool = poolmanager.pools[('http','localhost',12380)]
+                    pool = poolmanager.pools[('http','localhost',32180)]
                     return (pool.num_connections, pool.num_requests)
                 except:
                     raise SkipTest("can't get connection info")

tests/test_cmdline.py

@@ -32,7 +32,7 @@ def server_start(max_results = None, bulkdata_args = {}):
         max_results = max_results,
         bulkdata_args = bulkdata_args)
     test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+                                port = 32180, stoppable = False,
                                 fast_shutdown = True,
                                 force_traceback = False)
     test_server.start(blocking = False)
@@ -64,6 +64,7 @@ class TestCmdline(object):
         passing the given input.  Returns a tuple with the output and
         exit code"""
         # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
+        os.environ['NILMDB_URL'] = "http://localhost:32180/"
         class stdio_wrapper:
             def __init__(self, stdin, stdout, stderr):
                 self.io = (stdin, stdout, stderr)
@@ -174,7 +175,7 @@ class TestCmdline(object):
         self.fail("-u localhost:1 info")
         self.contain("error connecting to server")
 
-        self.ok("-u localhost:12380 info")
+        self.ok("-u localhost:32180 info")
         self.ok("info")
 
         # Duplicated arguments should fail, but this isn't implemented
@@ -192,6 +193,20 @@ class TestCmdline(object):
             self.fail("extract --start 2000-01-01 --start 2001-01-02")
             self.contain("duplicated argument")
 
+        # Verify that "help command" and "command --help" are identical
+        # for all commands.
+        self.fail("")
+        m = re.search(r"{(.*)}", self.captured)
+        for command in [""] + m.group(1).split(','):
+            self.ok(command + " --help")
+            cap1 = self.captured
+            self.ok("help " + command)
+            cap2 = self.captured
+            self.ok("help " + command + " asdf --url --zxcv -")
+            cap3 = self.captured
+            eq_(cap1, cap2)
+            eq_(cap2, cap3)
+
     def test_02_parsetime(self):
         os.environ['TZ'] = "America/New_York"
         test = datetime_tz.datetime_tz.now()
@@ -210,7 +225,7 @@ class TestCmdline(object):
 
     def test_03_info(self):
         self.ok("info")
-        self.contain("Server URL: http://localhost:12380/")
+        self.contain("Server URL: http://localhost:32180/")
         self.contain("Client version: " + nilmdb.__version__)
         self.contain("Server version: " + test_server.version)
         self.contain("Server database path")
@@ -418,7 +433,7 @@ class TestCmdline(object):
         # bad start time
         self.fail("insert --rate 120 --start 'whatever' /newton/prep /dev/null")
 
-    def test_07_detail(self):
+    def test_07_detail_extent(self):
         # Just count the number of lines, it's probably fine
         self.ok("list --detail")
         lines_(self.captured, 8)
@@ -463,6 +478,18 @@ class TestCmdline(object):
         lines_(self.captured, 2)
         self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
 
+        # Check --extent output
+        self.ok("list --extent")
+        lines_(self.captured, 6)
+
+        self.ok("list -E -T")
+        self.contain("  extent: 1332496800 -> 1332497159.991668")
+        self.contain("  extent: (no data)")
+
+        # Misc
+        self.fail("list --extent --start='23 Mar 2012 10:05:15.50'")
+        self.contain("--start and --end only make sense with --detail")
+
     def test_08_extract(self):
         # nonexistent stream
         self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")

tests/test_nilmdb.py

@@ -119,7 +119,7 @@ class TestBlockingServer(object):
 
         # Start web app on a custom port
         self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = True)
+                                    port = 32180, stoppable = True)
 
         # Run it
         event = threading.Event()
@@ -131,13 +131,13 @@ class TestBlockingServer(object):
             raise AssertionError("server didn't start in 10 seconds")
 
         # Send request to exit.
-        req = urlopen("http://127.0.0.1:12380/exit/", timeout = 1)
+        req = urlopen("http://127.0.0.1:32180/exit/", timeout = 1)
 
         # Wait for it
         thread.join()
 
 def geturl(path):
-    req = urlopen("http://127.0.0.1:12380" + path, timeout = 10)
+    req = urlopen("http://127.0.0.1:32180" + path, timeout = 10)
     return req.read()
 
 def getjson(path):
@@ -149,7 +149,7 @@ class TestServer(object):
         # Start web app on a custom port
         self.db = serializer_proxy(nilmdb.NilmDB)(testdb, sync=False)
         self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = False)
+                                    port = 32180, stoppable = False)
         self.server.start(blocking = False)
 
     def tearDown(self):