Print version at server startup

Replaces custom tools.allow_methods which didn't return the Allow: header.

nilmdb/client/client.py

@@ -52,12 +52,14 @@ class Client(object):
         as a dictionary."""
         return self.http.get("dbinfo")
 
-    def stream_list(self, path = None, layout = None):
+    def stream_list(self, path = None, layout = None, extent = False):
         params = {}
         if path is not None:
             params["path"] = path
         if layout is not None:
             params["layout"] = layout
+        if extent:
+            params["extent"] = 1
         return self.http.get("stream/list", params)
 
     def stream_get_metadata(self, path, keys = None):
@@ -220,7 +222,7 @@ class StreamInserter(object):
     # These are soft limits -- actual data might be rounded up.
     # We send when we have a certain amount of data queued, or
     # when a certain amount of time has passed since the last send.
-    _max_data = 1048576
+    _max_data = 2 * 1024 * 1024
     _max_time = 30
 
     # Delta to add to the final timestamp, if "end" wasn't given

nilmdb/client/httpclient.py

@@ -62,7 +62,8 @@ class HTTPClient(object):
         try:
             response = self.session.request(method, url,
                                             params = query_data,
-                                            data = body_data)
+                                            data = body_data,
+                                            stream = stream)
         except requests.RequestException as e:
             raise ServerError(status = "502 Error", url = url,
                               message = str(e.message))

nilmdb/cmdline/cmdline.py

@@ -6,13 +6,14 @@ from nilmdb.utils import datetime_tz
 import nilmdb.utils.time
 
 import sys
+import os
 import argparse
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
-                "remove", "destroy" ]
+subcommands = [ "help", "info", "create", "list", "metadata",
+                "insert", "extract", "remove", "destroy" ]
 
 # Import the subcommand modules
 subcmd_mods = {}
@@ -29,6 +30,8 @@ class Cmdline(object):
     def __init__(self, argv = None):
         self.argv = argv or sys.argv[1:]
         self.client = None
+        self.def_url = os.environ.get("NILMDB_URL", "http://localhost:12380")
+        self.subcmd = {}
 
     def arg_time(self, toparse):
         """Parse a time string argument"""
@@ -50,18 +53,17 @@ class Cmdline(object):
 
         group = self.parser.add_argument_group("Server")
         group.add_argument("-u", "--url", action="store",
-                           default="http://localhost:12380/",
+                           default=self.def_url,
                            help="NilmDB server URL (default: %(default)s)")
 
-        sub = self.parser.add_subparsers(title="Commands",
-                                         dest="command",
-                                         description="Specify --help after "
-                                         "the command for command-specific "
-                                         "options.")
+        sub = self.parser.add_subparsers(
+            title="Commands", dest="command",
+            description="Use 'help command' or 'command --help' for more "
+            "details on a particular command.")
 
         # Set up subcommands (defined in separate files)
         for cmd in subcommands:
-            subcmd_mods[cmd].setup(self, sub)
+            self.subcmd[cmd] = subcmd_mods[cmd].setup(self, sub)
 
     def die(self, formatstr, *args):
         fprintf(sys.stderr, formatstr + "\n", *args)
@@ -84,11 +86,13 @@ class Cmdline(object):
 
         self.client = nilmdb.Client(self.args.url)
 
-        # Make a test connection to make sure things work
-        try:
-            server_version = self.client.version()
-        except nilmdb.client.Error as e:
-            self.die("error connecting to server: %s", str(e))
+        # Make a test connection to make sure things work,
+        # unless the particular command requests that we don't.
+        if "no_test_connect" not in self.args:
+            try:
+                server_version = self.client.version()
+            except nilmdb.client.Error as e:
+                self.die("error connecting to server: %s", str(e))
 
         # Now dispatch client request to appropriate function.  Parser
         # should have ensured that we don't have any unknown commands

nilmdb/cmdline/create.py

@@ -26,6 +26,7 @@ Layout types are of the format: type_count
                        help="Path (in database) of new stream, e.g. /foo/bar")
     group.add_argument("layout",
                        help="Layout type for new stream, e.g. float32_8")
+    return cmd
 
 def cmd_create(self):
     """Create new stream"""

nilmdb/cmdline/destroy.py

@@ -16,6 +16,7 @@ def setup(self, sub):
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
                        help="Path of the stream to delete, e.g. /foo/bar")
+    return cmd
 
 def cmd_destroy(self):
     """Destroy stream"""

nilmdb/cmdline/extract.py

@@ -30,6 +30,7 @@ def setup(self, sub):
                        help="Show raw timestamps in annotated information")
     group.add_argument("-c", "--count", action="store_true",
                        help="Just output a count of matched data points")
+    return cmd
 
 def cmd_extract_verify(self):
     if self.args.start is not None and self.args.end is not None:

nilmdb/cmdline/help.py

@@ -0,0 +1,26 @@
+from nilmdb.utils.printf import *
+
+import argparse
+import sys
+
+def setup(self, sub):
+    cmd = sub.add_parser("help", help="Show detailed help for a command",
+                         description="""
+                         Show help for a command. 'help command' is
+                         the same as 'command --help'.
+                         """)
+    cmd.set_defaults(handler = cmd_help)
+    cmd.set_defaults(no_test_connect = True)
+    cmd.add_argument("command", nargs="?",
+                     help="Command to get help about")
+    cmd.add_argument("rest", nargs=argparse.REMAINDER,
+                     help=argparse.SUPPRESS)
+    return cmd
+
+def cmd_help(self):
+    if self.args.command in self.subcmd:
+        self.subcmd[self.args.command].print_help()
+    else:
+        self.parser.print_help()
+
+    return

nilmdb/cmdline/info.py

@@ -12,6 +12,7 @@ def setup(self, sub):
                          version.
                          """)
     cmd.set_defaults(handler = cmd_info)
+    return cmd
 
 def cmd_info(self):
     """Print info about the server"""

nilmdb/cmdline/insert.py

@@ -47,6 +47,7 @@ def setup(self, sub):
                        help="Path of stream, e.g. /foo/bar")
     group.add_argument("file", nargs="*", default=['-'],
                        help="File(s) to insert (default: - (stdin))")
+    return cmd
 
 def cmd_insert(self):
     # Find requested stream

nilmdb/cmdline/list.py

@@ -24,11 +24,13 @@ def setup(self, sub):
     group.add_argument("-l", "--layout", default="*",
                        help="Match only this stream layout")
 
+    group = cmd.add_argument_group("Interval extent")
+    group.add_argument("-E", "--extent", action="store_true",
+                       help="Show min/max timestamps in this stream")
+
     group = cmd.add_argument_group("Interval details")
     group.add_argument("-d", "--detail", action="store_true",
                        help="Show available data time intervals")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps in time intervals")
     group.add_argument("-s", "--start",
                        metavar="TIME", type=self.arg_time,
                        help="Starting timestamp (free-form, inclusive)")
@@ -36,6 +38,12 @@ def setup(self, sub):
                        metavar="TIME", type=self.arg_time,
                        help="Ending timestamp (free-form, noninclusive)")
 
+    group = cmd.add_argument_group("Misc options")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps in time intervals or extents")
+
+    return cmd
+
 def cmd_list_verify(self):
     # A hidden "path_positional" argument lets the user leave off the
     # "-p" when specifying the path.  Handle it here.
@@ -51,28 +59,38 @@ def cmd_list_verify(self):
         if self.args.start >= self.args.end:
             self.parser.error("start must precede end")
 
+    if self.args.start is not None or self.args.end is not None:
+        if not self.args.detail:
+            self.parser.error("--start and --end only make sense with --detail")
+
 def cmd_list(self):
     """List available streams"""
-    streams = self.client.stream_list()
+    streams = self.client.stream_list(extent = True)
 
     if self.args.timestamp_raw:
         time_string = repr
     else:
         time_string = nilmdb.utils.time.format_time
 
-    for (path, layout) in streams:
+    for (path, layout, extent_min, extent_max) in streams:
         if not (fnmatch.fnmatch(path, self.args.path) and
                 fnmatch.fnmatch(layout, self.args.layout)):
             continue
 
         printf("%s %s\n", path, layout)
-        if not self.args.detail:
-            continue
 
-        printed = False
-        for (start, end) in self.client.stream_intervals(path, self.args.start,
-                                                         self.args.end):
-            printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
-            printed = True
-        if not printed:
-            printf("  (no intervals)\n")
+        if self.args.extent:
+            if extent_min is None or extent_max is None:
+                printf("  extent: (no data)\n")
+            else:
+                printf("  extent: %s -> %s\n",
+                       time_string(extent_min), time_string(extent_max))
+
+        if self.args.detail:
+            printed = False
+            for (start, end) in self.client.stream_intervals(
+                path, self.args.start, self.args.end):
+                printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
+                printed = True
+            if not printed:
+                printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -26,6 +26,7 @@ def setup(self, sub):
     exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
                      help="Update metadata using provided "
                      "key=value pairs")
+    return cmd
 
 def cmd_metadata(self):
     """Manipulate metadata"""

nilmdb/cmdline/remove.py

@@ -23,6 +23,7 @@ def setup(self, sub):
     group = cmd.add_argument_group("Output format")
     group.add_argument("-c", "--count", action="store_true",
                        help="Output number of data points removed")
+    return cmd
 
 def cmd_remove(self):
     try:

nilmdb/scripts/nilmdb_server.py

@@ -25,6 +25,9 @@ def main():
                        default = os.path.join(os.getcwd(), "db"))
     group.add_argument('-q', '--quiet', help = 'Silence output',
                        action = 'store_true')
+    group.add_argument('-n', '--nosync', help = 'Use asynchronous '
+                       'commits for sqlite transactions',
+                       action = 'store_true', default = False)
 
     group = parser.add_argument_group("Debug options")
     group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and '
@@ -35,7 +38,8 @@ def main():
 
     # Create database object.  Needs to be serialized before passing
     # to the Server.
-    db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(args.database)
+    db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(args.database,
+                                                      sync = not args.nosync)
 
     # Configure the server
     if args.quiet:
@@ -49,6 +53,7 @@ def main():
 
     # Print info
     if not args.quiet:
+        print "Version: %s" % nilmdb.__version__
         print "Database: %s" % (os.path.realpath(args.database))
         if args.address == '0.0.0.0' or args.address == '::':
             host = socket.getfqdn()

nilmdb/server/bulkdata.py

@@ -28,7 +28,7 @@ except: # pragma: no cover
 table_cache_size = 16
 fd_cache_size = 16
 
-@nilmdb.utils.must_close(wrap_verify = True)
+@nilmdb.utils.must_close(wrap_verify = False)
 class BulkData(object):
     def __init__(self, basepath, **kwargs):
         self.basepath = basepath
@@ -171,7 +171,7 @@ class BulkData(object):
         ospath = os.path.join(self.root, *elements)
         return Table(ospath)
 
-@nilmdb.utils.must_close(wrap_verify = True)
+@nilmdb.utils.must_close(wrap_verify = False)
 class File(object):
     """Object representing a single file on disk.  Data can be appended,
     or the self.mmap handle can be used for random reads."""
@@ -210,14 +210,28 @@ class File(object):
             self.mmap.close()
         self._f.close()
 
-    def append(self, data):
+    def append(self, data): # pragma: no cover (below version used instead)
         # Write data, flush it, and resize our mmap accordingly
         self._f.write(data)
         self._f.flush()
         self.size += len(data)
         self._mmap_reopen()
 
-@nilmdb.utils.must_close(wrap_verify = True)
+    def append_pack_iter(self, count, packer, dataiter):
+        # An optimized verison of append, to avoid flushing the file
+        # and resizing the mmap after each data point.
+        try:
+            rows = []
+            for i in xrange(count):
+                row = dataiter.next()
+                rows.append(packer(*row))
+            self._f.write("".join(rows))
+        finally:
+            self._f.flush()
+            self.size = self._f.tell()
+            self._mmap_reopen()
+
+@nilmdb.utils.must_close(wrap_verify = False)
 class Table(object):
     """Tools to help access a single table (data at a specific OS path)."""
     # See design.md for design details
@@ -351,9 +365,7 @@ class Table(object):
             f = self.file_open(subdir, fname)
 
             # Write the data
-            for i in xrange(count):
-                row = dataiter.next()
-                f.append(self.packer.pack(*row))
+            f.append_pack_iter(count, self.packer.pack, dataiter)
             remaining -= count
             self.nrows += count
 

nilmdb/server/layout.pyx

@@ -4,7 +4,6 @@ import time
 import sys
 import inspect
 import cStringIO
-import numpy as np
 
 cdef enum:
     max_value_count = 64
@@ -42,10 +41,12 @@ class Layout:
 
         if datatype == 'uint16':
             self.parse = self.parse_uint16
-            self.format = self.format_uint16
+            self.format_str = "%.6f" + " %d" * self.count
+            self.format = self.format_generic
         elif datatype == 'float32' or datatype == 'float64':
             self.parse = self.parse_float64
-            self.format = self.format_float64
+            self.format_str = "%.6f" + " %f" * self.count
+            self.format = self.format_generic
         else:
             raise KeyError("invalid type")
 
@@ -57,15 +58,15 @@ class Layout:
         cdef double ts
         # Return doubles even in float32 case, since they're going into
         # a Python array which would upconvert to double anyway.
-        result = []
+        result = [0] * (self.count + 1)
         cdef char *end
         ts = libc.stdlib.strtod(text, &end)
         if end == text:
             raise ValueError("bad timestamp")
-        result.append(ts)
+        result[0] = ts
         for n in range(self.count):
             text = end
-            result.append(libc.stdlib.strtod(text, &end))
+            result[n+1] = libc.stdlib.strtod(text, &end)
             if end == text:
                 raise ValueError("wrong number of values")
         n = 0
@@ -79,18 +80,18 @@ class Layout:
         cdef int n
         cdef double ts
         cdef int v
-        result = []
         cdef char *end
+        result = [0] * (self.count + 1)
         ts = libc.stdlib.strtod(text, &end)
         if end == text:
             raise ValueError("bad timestamp")
-        result.append(ts)
+        result[0] = ts
         for n in range(self.count):
             text = end
             v = libc.stdlib.strtol(text, &end, 10)
             if v < 0 or v > 65535:
                 raise ValueError("value out of range")
-            result.append(v)
+            result[n+1] = v
             if end == text:
                 raise ValueError("wrong number of values")
         n = 0
@@ -101,25 +102,12 @@ class Layout:
         return (ts, result)
 
     # Formatters
-    def format_float64(self, d):
+    def format_generic(self, d):
         n = len(d) - 1
         if n != self.count:
             raise ValueError("wrong number of values for layout type: "
                              "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %f" % d[i+1]
-        return s + "\n"
-
-    def format_uint16(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %d" % d[i+1]
-        return s + "\n"
+        return (self.format_str % tuple(d)) + "\n"
 
 # Get a layout by name
 def get_named(typestring):

nilmdb/server/nilmdb.py

@@ -269,28 +269,39 @@ class NilmDB(object):
 
         return
 
-    def stream_list(self, path = None, layout = None):
-        """Return list of [path, layout] lists of all streams
-        in the database.
+    def stream_list(self, path = None, layout = None, extent = False):
+        """Return list of lists of all streams in the database.
 
         If path is specified, include only streams with a path that
         matches the given string.
 
         If layout is specified, include only streams with a layout
         that matches the given string.
-        """
-        where = "WHERE 1=1"
-        params = ()
-        if layout:
-            where += " AND layout=?"
-            params += (layout,)
-        if path:
-            where += " AND path=?"
-            params += (path,)
-        result = self.con.execute("SELECT path, layout "
-                                  "FROM streams " + where, params).fetchall()
 
-        return sorted(list(x) for x in result)
+        If extent = False, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extent = True, returns a list of lists containing the
+        path, layout, and min/max extent of the data:
+        [ path, layout, extent_min, extent_max ]
+        """
+        params = ()
+        query = "SELECT streams.path, streams.layout"
+        if extent:
+            query += ", min(ranges.start_time), max(ranges.end_time)"
+        query += " FROM streams"
+        if extent:
+            query += " LEFT JOIN ranges ON streams.id = ranges.stream_id"
+        query += " WHERE 1=1"
+        if layout is not None:
+            query += " AND streams.layout=?"
+            params += (layout,)
+        if path is not None:
+            query += " AND streams.path=?"
+            params += (path,)
+        query += " GROUP BY streams.id ORDER BY streams.path"
+        result = self.con.execute(query, params).fetchall()
+        return [ list(x) for x in result ]
 
     def stream_intervals(self, path, start = None, end = None):
         """

nilmdb/server/server.py

@@ -71,16 +71,51 @@ def exception_to_httperror(*expected):
     # care of that.
     return decorator.decorator(wrapper)
 
-# Custom Cherrypy tools
-def allow_methods(methods):
-    method = cherrypy.request.method.upper()
-    if method not in methods:
-        if method in cherrypy.request.methods_with_bodies:
-            cherrypy.request.body.read()
-        allowed = ', '.join(methods)
-        cherrypy.response.headers['Allow'] = allowed
-        raise cherrypy.HTTPError(405, method + " not allowed; use " + allowed)
-cherrypy.tools.allow_methods = cherrypy.Tool('before_handler', allow_methods)
+# Custom CherryPy tools
+
+def CORS_allow(methods):
+    """This does several things:
+
+    Handles CORS preflight requests.
+    Adds Allow: header to all requests.
+    Raise 405 if request.method not in method.
+
+    It is similar to cherrypy.tools.allow, with the CORS stuff added.
+    """
+    request = cherrypy.request.headers
+    response = cherrypy.response.headers
+
+    if not isinstance(methods, (tuple, list)): # pragma: no cover
+        methods = [ methods ]
+    methods = [ m.upper() for m in methods if m ]
+    if not methods: # pragma: no cover
+        methods = [ 'GET', 'HEAD' ]
+    elif 'GET' in methods and 'HEAD' not in methods: # pragma: no cover
+        methods.append('HEAD')
+    response['Allow'] = ', '.join(methods)
+
+    # Allow all origins
+    if 'Origin' in request:
+        response['Access-Control-Allow-Origin'] = request['Origin']
+
+    # If it's a CORS request, send response.
+    request_method = request.get("Access-Control-Request-Method", None)
+    request_headers = request.get("Access-Control-Request-Headers", None)
+    if (cherrypy.request.method == "OPTIONS" and
+        request_method and request_headers):
+        response['Access-Control-Allow-Headers'] = request_headers
+        response['Access-Control-Allow-Methods'] = ', '.join(methods)
+        # Try to stop further processing and return a 200 OK
+        cherrypy.response.status = "200 OK"
+        cherrypy.response.body = ""
+        cherrypy.request.handler = lambda: ""
+        return
+
+    # Reject methods that were not explicitly allowed
+    if cherrypy.request.method not in methods:
+        raise cherrypy.HTTPError(405)
+
+cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
 
 # CherryPy apps
 class Root(NilmApp):
@@ -121,20 +156,28 @@ class Stream(NilmApp):
 
     # /stream/list
     # /stream/list?layout=PrepData
-    # /stream/list?path=/newton/prep
+    # /stream/list?path=/newton/prep&extent=1
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def list(self, path = None, layout = None):
+    def list(self, path = None, layout = None, extent = None):
         """List all streams in the database.  With optional path or
         layout parameter, just list streams that match the given path
-        or layout"""
-        return self.db.stream_list(path, layout)
+        or layout.
+
+        If extent is not given, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extent is provided, returns a list of lists containing the
+        path, layout, and min/max extent of the data:
+        [ path, layout, extent_min, extent_max ]
+        """
+        return self.db.stream_list(path, layout, bool(extent))
 
     # /stream/create?path=/newton/prep&layout=PrepData
     @cherrypy.expose
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError, ValueError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def create(self, path, layout):
         """Create a new stream in the database.  Provide path
         and one of the nilmdb.layout.layouts keys.
@@ -145,7 +188,7 @@ class Stream(NilmApp):
     @cherrypy.expose
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def destroy(self, path):
         """Delete a stream and its associated data."""
         return self.db.stream_destroy(path)
@@ -178,7 +221,7 @@ class Stream(NilmApp):
     @cherrypy.expose
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError, LookupError, TypeError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def set_metadata(self, path, data):
         """Set metadata for the named stream, replacing any
         existing metadata.  Data should be a json-encoded
@@ -190,7 +233,7 @@ class Stream(NilmApp):
     @cherrypy.expose
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError, LookupError, TypeError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def update_metadata(self, path, data):
         """Update metadata for the named stream.  Data
         should be a json-encoded dictionary"""
@@ -200,7 +243,7 @@ class Stream(NilmApp):
     # /stream/insert?path=/newton/prep
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    @cherrypy.tools.allow_methods(methods = ["PUT"])
+    @cherrypy.tools.CORS_allow(methods = ["PUT"])
     def insert(self, path, start, end):
         """
         Insert new data into the database.  Provide textual data
@@ -256,7 +299,7 @@ class Stream(NilmApp):
     @cherrypy.expose
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def remove(self, path, start = None, end = None):
         """
         Remove data from the backend database.  Removes all data in
@@ -400,7 +443,7 @@ class Server(object):
             'server.socket_host': host,
             'server.socket_port': port,
             'engine.autoreload_on': False,
-            'server.max_request_body_size': 4*1024*1024,
+            'server.max_request_body_size': 8*1024*1024,
             })
         if self.embedded:
             cherrypy.config.update({ 'environment': 'embedded' })
@@ -411,16 +454,14 @@ class Server(object):
             'error_page.default': self.json_error_page,
             })
 
-        # Send a permissive Access-Control-Allow-Origin (CORS) header
-        # with all responses so that browsers can send cross-domain
-        # requests to this server.
-        app_config.update({ 'response.headers.Access-Control-Allow-Origin':
-                            '*' })
+        # Some default headers to just help identify that things are working
+        app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
 
-        # Only allow GET and HEAD by default.  Individual handlers
-        # can override.
-        app_config.update({ 'tools.allow_methods.on': True,
-                            'tools.allow_methods.methods': ['GET', 'HEAD'] })
+        # Set up Cross-Origin Resource Sharing (CORS) handler so we
+        # can correctly respond to browsers' CORS preflight requests.
+        # This also limits verbs to GET and HEAD by default.
+        app_config.update({ 'tools.CORS_allow.on': True,
+                            'tools.CORS_allow.methods': ['GET', 'HEAD'] })
 
         # Send tracebacks in error responses.  They're hidden by the
         # error_page function for client errors (code 400-499).

tests/test_client.py

@@ -20,11 +20,12 @@ import unittest
 import warnings
 import resource
 import time
+import re
 
 from testutil.helpers import *
 
 testdb = "tests/client-testdb"
-testurl = "http://localhost:12380/"
+testurl = "http://localhost:32180/"
 
 def setup_module():
     global test_server, test_db
@@ -34,7 +35,7 @@ def setup_module():
     # Start web app on a custom port
     test_db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(testdb, sync = False)
     test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+                                port = 32180, stoppable = False,
                                 fast_shutdown = True,
                                 force_traceback = False)
     test_server.start(blocking = False)
@@ -54,20 +55,14 @@ class TestClient(object):
             client.version()
         client.close()
 
-        # Trigger same error with a PUT request
-        client = nilmdb.Client(url = "http://localhost:1/")
-        with assert_raises(nilmdb.client.ServerError):
-            client.version()
-        client.close()
-
         # Then a fake URL on a real host
-        client = nilmdb.Client(url = "http://localhost:12380/fake/")
+        client = nilmdb.Client(url = "http://localhost:32180/fake/")
         with assert_raises(nilmdb.client.ClientError):
             client.version()
         client.close()
 
         # Now a real URL with no http:// prefix
-        client = nilmdb.Client(url = "localhost:12380")
+        client = nilmdb.Client(url = "localhost:32180")
         version = client.version()
         client.close()
 
@@ -172,6 +167,10 @@ class TestClient(object):
     def test_client_04_insert(self):
         client = nilmdb.Client(url = testurl)
 
+        # Limit _max_data to 1 MB, since our test file is 1.5 MB
+        old_max_data = nilmdb.client.client.StreamInserter._max_data
+        nilmdb.client.client.StreamInserter._max_data = 1 * 1024 * 1024
+
         datetime_tz.localtz_set("America/New_York")
 
         testfile = "tests/data/prep-20120323T1000"
@@ -244,8 +243,9 @@ class TestClient(object):
         in_("400 Bad Request", str(e.exception))
         # Client chunks the input, so the exact timestamp here might change
         # if the chunk positions change.
-        in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
-            str(e.exception))
+        assert(re.search("Data timestamp 13325[0-9]+\.[0-9]+ "
+                         ">= end time 1332511201.0", str(e.exception))
+               is not None)
 
         # Now do the real load
         data = timestamper.TimestamperRate(testfile, start, 120)
@@ -264,6 +264,7 @@ class TestClient(object):
         in_("400 Bad Request", str(e.exception))
         in_("verlap", str(e.exception))
 
+        nilmdb.client.client.StreamInserter._max_data = old_max_data
         client.close()
 
     def test_client_05_extractremove(self):
@@ -353,12 +354,6 @@ class TestClient(object):
             raise AssertionError("/stream/extract is not text/plain:\n" +
                                  headers())
 
-        # Make sure Access-Control-Allow-Origin gets set
-        if "access-control-allow-origin: " not in headers():
-            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
-                                 "header in /stream/extract response:\n" +
-                                 headers())
-
         client.close()
 
     def test_client_08_unicode(self):
@@ -577,7 +572,7 @@ class TestClient(object):
             def connections():
                 try:
                     poolmanager = c.http._last_response.connection.poolmanager
-                    pool = poolmanager.pools[('http','localhost',12380)]
+                    pool = poolmanager.pools[('http','localhost',32180)]
                     return (pool.num_connections, pool.num_requests)
                 except:
                     raise SkipTest("can't get connection info")

tests/test_cmdline.py

@@ -11,12 +11,7 @@ from nose.tools import assert_raises
 import itertools
 import os
 import re
-import shutil
 import sys
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
 import StringIO
 import shlex
 
@@ -32,7 +27,7 @@ def server_start(max_results = None, bulkdata_args = {}):
         max_results = max_results,
         bulkdata_args = bulkdata_args)
     test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+                                port = 32180, stoppable = False,
                                 fast_shutdown = True,
                                 force_traceback = False)
     test_server.start(blocking = False)
@@ -64,6 +59,7 @@ class TestCmdline(object):
         passing the given input.  Returns a tuple with the output and
         exit code"""
         # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
+        os.environ['NILMDB_URL'] = "http://localhost:32180/"
         class stdio_wrapper:
             def __init__(self, stdin, stdout, stderr):
                 self.io = (stdin, stdout, stderr)
@@ -174,7 +170,7 @@ class TestCmdline(object):
         self.fail("-u localhost:1 info")
         self.contain("error connecting to server")
 
-        self.ok("-u localhost:12380 info")
+        self.ok("-u localhost:32180 info")
         self.ok("info")
 
         # Duplicated arguments should fail, but this isn't implemented
@@ -192,6 +188,20 @@ class TestCmdline(object):
             self.fail("extract --start 2000-01-01 --start 2001-01-02")
             self.contain("duplicated argument")
 
+        # Verify that "help command" and "command --help" are identical
+        # for all commands.
+        self.fail("")
+        m = re.search(r"{(.*)}", self.captured)
+        for command in [""] + m.group(1).split(','):
+            self.ok(command + " --help")
+            cap1 = self.captured
+            self.ok("help " + command)
+            cap2 = self.captured
+            self.ok("help " + command + " asdf --url --zxcv -")
+            cap3 = self.captured
+            eq_(cap1, cap2)
+            eq_(cap2, cap3)
+
     def test_02_parsetime(self):
         os.environ['TZ'] = "America/New_York"
         test = datetime_tz.datetime_tz.now()
@@ -210,7 +220,7 @@ class TestCmdline(object):
 
     def test_03_info(self):
         self.ok("info")
-        self.contain("Server URL: http://localhost:12380/")
+        self.contain("Server URL: http://localhost:32180/")
         self.contain("Client version: " + nilmdb.__version__)
         self.contain("Server version: " + test_server.version)
         self.contain("Server database path")
@@ -418,7 +428,7 @@ class TestCmdline(object):
         # bad start time
         self.fail("insert --rate 120 --start 'whatever' /newton/prep /dev/null")
 
-    def test_07_detail(self):
+    def test_07_detail_extent(self):
         # Just count the number of lines, it's probably fine
         self.ok("list --detail")
         lines_(self.captured, 8)
@@ -463,6 +473,18 @@ class TestCmdline(object):
         lines_(self.captured, 2)
         self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
 
+        # Check --extent output
+        self.ok("list --extent")
+        lines_(self.captured, 6)
+
+        self.ok("list -E -T")
+        self.contain("  extent: 1332496800 -> 1332497159.991668")
+        self.contain("  extent: (no data)")
+
+        # Misc
+        self.fail("list --extent --start='23 Mar 2012 10:05:15.50'")
+        self.contain("--start and --end only make sense with --detail")
+
     def test_08_extract(self):
         # nonexistent stream
         self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")

tests/test_layout.py

@@ -9,14 +9,7 @@ from nose.tools import assert_raises
 import distutils.version
 import itertools
 import os
-import shutil
 import sys
-import cherrypy
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
-import cStringIO
 import random
 import unittest
 
@@ -246,7 +239,7 @@ class TestLayoutSpeed:
                 parser = Parser(layout)
                 formatter = Formatter(layout)
                 parser.parse(data)
-                data = formatter.format(parser.data)
+                formatter.format(parser.data)
             elapsed = time.time() - start
             printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
                    layout,
@@ -264,3 +257,8 @@ class TestLayoutSpeed:
             return [ sprintf("%d", random.randint(0,65535))
                      for x in range(10) ]
         do_speedtest("uint16_10", datagen)
+
+        def datagen():
+            return [ sprintf("%d", random.randint(0,65535))
+                     for x in range(6) ]
+        do_speedtest("uint16_6", datagen)

tests/test_mustclose.py

@@ -34,6 +34,10 @@ class Bar:
     def __del__(self):
         fprintf(err, "Deleting\n")
 
+    @classmethod
+    def baz(self):
+        fprintf(err, "Baz\n")
+
     def close(self):
         fprintf(err, "Closing\n")
 

tests/test_nilmdb.py

@@ -6,15 +6,13 @@ import distutils.version
 import simplejson as json
 import itertools
 import os
-import shutil
 import sys
-import cherrypy
 import threading
 import urllib2
 from urllib2 import urlopen, HTTPError
-import Queue
 import cStringIO
 import time
+import requests
 
 from nilmdb.utils import serializer_proxy
 
@@ -119,7 +117,7 @@ class TestBlockingServer(object):
 
         # Start web app on a custom port
         self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = True)
+                                    port = 32180, stoppable = True)
 
         # Run it
         event = threading.Event()
@@ -131,13 +129,13 @@ class TestBlockingServer(object):
             raise AssertionError("server didn't start in 10 seconds")
 
         # Send request to exit.
-        req = urlopen("http://127.0.0.1:12380/exit/", timeout = 1)
+        req = urlopen("http://127.0.0.1:32180/exit/", timeout = 1)
 
         # Wait for it
         thread.join()
 
 def geturl(path):
-    req = urlopen("http://127.0.0.1:12380" + path, timeout = 10)
+    req = urlopen("http://127.0.0.1:32180" + path, timeout = 10)
     return req.read()
 
 def getjson(path):
@@ -149,7 +147,7 @@ class TestServer(object):
         # Start web app on a custom port
         self.db = serializer_proxy(nilmdb.NilmDB)(testdb, sync=False)
         self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = False)
+                                    port = 32180, stoppable = False)
         self.server.start(blocking = False)
 
     def tearDown(self):
@@ -208,3 +206,34 @@ class TestServer(object):
         data = getjson("/stream/get_metadata?path=/newton/prep"
                        "&key=foo")
         eq_(data, {'foo': None})
+
+    def test_cors_headers(self):
+        # Test that CORS headers are being set correctly
+
+        # Normal GET should send simple response
+        url = "http://127.0.0.1:32180/stream/list"
+        r = requests.get(url, headers = { "Origin": "http://google.com/" })
+        eq_(r.status_code, 200)
+        if "access-control-allow-origin" not in r.headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-origin"], "http://google.com/")
+
+        # OPTIONS without CORS preflight headers should result in 405
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            })
+        eq_(r.status_code, 405)
+
+        # OPTIONS with preflight headers should give preflight response
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            "Access-Control-Request-Method": "POST",
+            "Access-Control-Request-Headers": "X-Custom",
+            })
+        eq_(r.status_code, 200)
+        if "access-control-allow-origin" not in r.headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-methods"], "GET, HEAD")
+        eq_(r.headers["access-control-allow-headers"], "X-Custom")