Merge remote-tracking branch 'origin/packaging'

datetime_tz isn't readily available, so it's a lot easier to just
package it within the nilmdb tree.

.coveragerc

@@ -7,3 +7,4 @@
 exclude_lines =
 	pragma: no cover
 	if 0:
+omit = nilmdb/utils/datetime_tz*

.gitignore

@@ -1,7 +1,23 @@
-db/
+# Tests
 tests/*testdb/
 .coverage
+db/
+
+# Compiled / cythonized files
+docs/*.html
+build/
 *.pyc
-design.html
+nilmdb/server/interval.c
+nilmdb/server/interval.so
+nilmdb/server/layout.c
+nilmdb/server/layout.so
+nilmdb/server/rbtree.c
+nilmdb/server/rbtree.so
+
+# Setup junk
+dist/
+nilmdb.egg-info/
+
+# Misc
 timeit*out
 

Makefile

@@ -1,22 +1,11 @@
 all: test
 
-tool:
-	python nilmtool.py  --help
-	python nilmtool.py list --help
-	python nilmtool.py -u asfdadsf list
-
 lint:
 	pylint -f parseable nilmdb
 
-%.html: %.md
-	pandoc -s $< > $@
-
 test:
 	python runtests.py
 
-profile:
-	python runtests.py --with-profile
-
 clean::
 	find . -name '*pyc' | xargs rm -f
 	rm -f .coverage

README.txt

@@ -1,3 +1,10 @@
-sudo apt-get install python2.7 python-cherrypy3 python-decorator python-nose python-coverage
+nilmdb: Non-Intrusive Load Monitor Database
-sudo apt-get install cython # 0.17.1-1 or newer
+by Jim Paris <jim@jtan.com>
 
+Prerequisites:
+
+  sudo apt-get install python2.7 python-cherrypy3 python-decorator python-nose python-coverage python-setuptools
+
+Install:
+
+  python setup.py install

TODO

@@ -1,5 +0,0 @@
--- Clean up error responses.  Specifically I'd like to be able to add
-   json-formatted data to OverflowError and DB parsing errors.  It
-   seems like subclassing cherrypy.HTTPError and overriding
-   set_response is the best thing to do -- it would let me get rid
-   of the _be_ie_unfriendly and other hacks in the server.

docs/Makefile

@@ -0,0 +1,9 @@
+ALL_DOCS = $(wildcard *.md)
+
+all: $(ALL_DOCS:.md=.html)
+
+%.html: %.md
+	pandoc -s $< > $@
+
+clean:
+	rm -f *.html

docs/TODO.md

@@ -0,0 +1,5 @@
+- Documentation
+
+- Machine-readable information in OverflowError, parser errors.
+  Maybe subclass `cherrypy.HTTPError` and override `set_response`
+  to add another JSON field?

nilmdb/__init__.py

@@ -1,12 +1,4 @@
 """Main NilmDB import"""
 
-from .nilmdb import NilmDB
+from server import NilmDB, Server
-from .server import Server
+from client import Client
-from .client import Client
-
-import pyximport; pyximport.install()
-import layout
-import interval
-
-import cmdline
-

nilmdb/client/__init__.py

@@ -0,0 +1,4 @@
+"""nilmdb.client"""
+
+from .client import Client
+from .errors import *

nilmdb/client/client.py

@@ -2,7 +2,9 @@
 
 """Class for performing HTTP client requests via libcurl"""
 
-from __future__ import absolute_import
+import nilmdb
+import nilmdb.utils
+import nilmdb.client.httpclient
 from nilmdb.utils.printf import *
 
 import time
@@ -12,12 +14,6 @@ import os
 import simplejson as json
 import itertools
 
-import nilmdb.utils
-import nilmdb.httpclient
-
-# Other functions expect to see these in the nilmdb.client namespace
-from nilmdb.httpclient import ClientError, ServerError, Error
-
 version = "1.0"
 
 def float_to_string(f):
@@ -30,7 +26,7 @@ class Client(object):
     client_version = version
 
     def __init__(self, url):
-        self.http = nilmdb.httpclient.HTTPClient(url)
+        self.http = nilmdb.client.httpclient.HTTPClient(url)
 
     def _json_param(self, data):
         """Return compact json-encoded version of parameter"""

nilmdb/client/errors.py

@@ -0,0 +1,33 @@
+"""HTTP client errors"""
+
+from nilmdb.utils.printf import *
+
+class Error(Exception):
+    """Base exception for both ClientError and ServerError responses"""
+    def __init__(self,
+                 status = "Unspecified error",
+                 message = None,
+                 url = None,
+                 traceback = None):
+        Exception.__init__(self, status)
+        self.status = status     # e.g. "400 Bad Request"
+        self.message = message   # textual message from the server
+        self.url = url           # URL we were requesting
+        self.traceback = traceback # server traceback, if available
+    def _format_error(self, show_url):
+        s = sprintf("[%s]", self.status)
+        if self.message:
+            s += sprintf(" %s", self.message)
+        if show_url and self.url: # pragma: no cover
+            s += sprintf(" (%s)", self.url)
+        if self.traceback: # pragma: no cover
+            s += sprintf("\nServer traceback:\n%s", self.traceback)
+        return s
+    def __str__(self):
+        return self._format_error(show_url = False)
+    def __repr__(self): # pragma: no cover
+        return self._format_error(show_url = True)
+class ClientError(Error):
+    pass
+class ServerError(Error):
+    pass

nilmdb/client/httpclient.py

@@ -1,8 +1,9 @@
 """HTTP client library"""
 
-from __future__ import absolute_import
+import nilmdb
-from nilmdb.utils.printf import *
 import nilmdb.utils
+from nilmdb.utils.printf import *
+from nilmdb.client.errors import *
 
 import time
 import sys
@@ -13,39 +14,6 @@ import urlparse
 import pycurl
 import cStringIO
 
-class Error(Exception):
-    """Base exception for both ClientError and ServerError responses"""
-    def __init__(self,
-                 status = "Unspecified error",
-                 message = None,
-                 url = None,
-                 traceback = None):
-        Exception.__init__(self, status)
-        self.status = status     # e.g. "400 Bad Request"
-        self.message = message   # textual message from the server
-        self.url = url           # URL we were requesting
-        self.traceback = traceback # server traceback, if available
-    def __str__(self):
-        s = sprintf("[%s]", self.status)
-        if self.message:
-            s += sprintf(" %s", self.message)
-        if self.traceback: # pragma: no cover
-            s += sprintf("\nServer traceback:\n%s", self.traceback)
-        return s
-    def __repr__(self): # pragma: no cover
-        s = sprintf("[%s]", self.status)
-        if self.message:
-            s += sprintf(" %s", self.message)
-        if self.url:
-            s += sprintf(" (%s)", self.url)
-        if self.traceback:
-            s += sprintf("\nServer traceback:\n%s", self.traceback)
-        return s
-class ClientError(Error):
-    pass
-class ServerError(Error):
-    pass
-
 class HTTPClient(object):
     """Class to manage and perform HTTP requests from the client"""
     def __init__(self, baseurl = ""):
@@ -119,13 +87,14 @@ class HTTPClient(object):
             self.curl.setopt(pycurl.WRITEFUNCTION, callback)
             self.curl.perform()
         try:
-            for i in nilmdb.utils.Iteratorizer(func):
+            with nilmdb.utils.Iteratorizer(func, curl_hack = True) as it:
-                if self._status == 200:
+                for i in it:
-                    # If we had a 200 response, yield the data to the caller.
+                    if self._status == 200:
-                    yield i
+                        # If we had a 200 response, yield the data to caller.
-                else:
+                        yield i
-                    # Otherwise, collect it into an error string.
+                    else:
-                    error_body += i
+                        # Otherwise, collect it into an error string.
+                        error_body += i
         except pycurl.error as e:
             raise ServerError(status = "502 Error",
                               url = self.url,
@@ -195,9 +164,9 @@ class HTTPClient(object):
 
     def put(self, url, postdata, params = None, retjson = True):
         """Simple PUT"""
+        self.curl.setopt(pycurl.UPLOAD, 1)
         self._setup_url(url, params)
         data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.UPLOAD, 1)
         self.curl.setopt(pycurl.READFUNCTION, data.read)
         return self._doreq(url, params, retjson)
 
@@ -223,8 +192,8 @@ class HTTPClient(object):
 
     def put_gen(self, url, postdata, params = None, retjson = True):
         """Simple PUT, returning a generator"""
+        self.curl.setopt(pycurl.UPLOAD, 1)
         self._setup_url(url, params)
         data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.UPLOAD, 1)
         self.curl.setopt(pycurl.READFUNCTION, data.read)
         return self._doreq_gen(url, params, retjson)

nilmdb/cmdline/__init__.py

@@ -1 +1,3 @@
+"""nilmdb.cmdline"""
+
 from .cmdline import Cmdline

nilmdb/cmdline/cmdline.py

@@ -1,10 +1,9 @@
 """Command line client functionality"""
 
-from __future__ import absolute_import
+import nilmdb
 from nilmdb.utils.printf import *
-import nilmdb.client
+from nilmdb.utils import datetime_tz
 
-import datetime_tz
 import dateutil.parser
 import sys
 import re
@@ -49,10 +48,10 @@ class Cmdline(object):
         If the string doesn't contain a timestamp, the current local
         timezone is assumed (e.g. from the TZ env var).
         """
-        # If string doesn't contain at least 6 digits, consider it
+        # If string isn't "now" and doesn't contain at least 4 digits,
-        # invalid.  smartparse might otherwise accept empty strings
+        # consider it invalid.  smartparse might otherwise accept
-        # and strings with just separators.
+        # empty strings and strings with just separators.
-        if len(re.findall(r"\d", toparse)) < 6:
+        if toparse != "now" and len(re.findall(r"\d", toparse)) < 4:
             raise ValueError("not enough digits for a timestamp")
 
         # Try to just parse the time as given

nilmdb/cmdline/create.py

@@ -1,17 +1,27 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
+import textwrap
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form
+from argparse import RawDescriptionHelpFormatter as raw_form
 
 def setup(self, sub):
     cmd = sub.add_parser("create", help="Create a new stream",
-                         formatter_class = def_form,
+                         formatter_class = raw_form,
                          description="""
-                         Create a new empty stream at the
+Create a new empty stream at the specified path and with the specified
-                         specified path and with the specifed
+layout type.
-                         layout type.
+
-                         """)
+Layout types are of the format: type_count
+
+  'type' is a data type like 'float32', 'float64', 'uint16', 'int32', etc.
+
+  'count' is the number of columns of this type.
+
+  For example, 'float32_8' means the data for this stream has 8 columns of
+  32-bit floating point values.
+""")
     cmd.set_defaults(handler = cmd_create)
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",

nilmdb/cmdline/destroy.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form

nilmdb/cmdline/extract.py

@@ -1,6 +1,6 @@
-from __future__ import absolute_import
 from __future__ import print_function
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 import sys
 
@@ -28,6 +28,8 @@ def setup(self, sub):
     group.add_argument("-a", "--annotate", action="store_true",
                        help="Include comments with some information "
                        "about the stream")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps in annotated information")
     group.add_argument("-c", "--count", action="store_true",
                        help="Just output a count of matched data points")
 
@@ -42,11 +44,16 @@ def cmd_extract(self):
         self.die("error getting stream info for path %s", self.args.path)
     layout = streams[0][1]
 
+    if self.args.timestamp_raw:
+        time_string = repr
+    else:
+        time_string = self.time_string
+
     if self.args.annotate:
         printf("# path: %s\n", self.args.path)
         printf("# layout: %s\n", layout)
-        printf("# start: %s\n", self.time_string(self.args.start))
+        printf("# start: %s\n", time_string(self.args.start))
-        printf("# end: %s\n", self.time_string(self.args.end))
+        printf("# end: %s\n", time_string(self.args.end))
 
     printed = False
     for dataline in self.client.stream_extract(self.args.path,

nilmdb/cmdline/info.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form

nilmdb/cmdline/insert.py

@@ -1,7 +1,7 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
-import nilmdb.timestamper
+import nilmdb.utils.timestamper as timestamper
 
 import sys
 
@@ -69,7 +69,7 @@ def cmd_insert(self):
 
         # Build a timestamper for this file
         if self.args.none:
-            ts = nilmdb.timestamper.TimestamperNull(infile)
+            ts = timestamper.TimestamperNull(infile)
         else:
             if self.args.start:
                 start = self.args.start
@@ -84,7 +84,7 @@ def cmd_insert(self):
                 self.die("error: --rate is needed, but was not specified")
             rate = self.args.rate
 
-            ts = nilmdb.timestamper.TimestamperRate(infile, start, rate)
+            ts = timestamper.TimestamperRate(infile, start, rate)
 
         # Print info
         if not self.args.quiet:

nilmdb/cmdline/list.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 
 import fnmatch
@@ -28,6 +28,8 @@ def setup(self, sub):
     group = cmd.add_argument_group("Interval details")
     group.add_argument("-d", "--detail", action="store_true",
                        help="Show available data time intervals")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps in time intervals")
     group.add_argument("-s", "--start",
                        metavar="TIME", type=self.arg_time,
                        help="Starting timestamp (free-form, inclusive)")
@@ -53,6 +55,12 @@ def cmd_list_verify(self):
 def cmd_list(self):
     """List available streams"""
     streams = self.client.stream_list()
+
+    if self.args.timestamp_raw:
+        time_string = repr
+    else:
+        time_string = self.time_string
+
     for (path, layout) in streams:
         if not (fnmatch.fnmatch(path, self.args.path) and
                 fnmatch.fnmatch(layout, self.args.layout)):
@@ -65,9 +73,7 @@ def cmd_list(self):
         printed = False
         for (start, end) in self.client.stream_intervals(path, self.args.start,
                                                          self.args.end):
-            printf("  [ %s -> %s ]\n",
+            printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
-                   self.time_string(start),
-                   self.time_string(end))
             printed = True
         if not printed:
             printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 
 def setup(self, sub):

nilmdb/cmdline/remove.py

@@ -1,6 +1,5 @@
-from __future__ import absolute_import
-from __future__ import print_function
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 import sys
 

nilmdb/server/__init__.py

@@ -0,0 +1,15 @@
+"""nilmdb.server"""
+
+# Try to set up pyximport to automatically rebuild Cython modules.  If
+# this doesn't work, it's OK, as long as the modules were built externally.
+# (e.g. python setup.py build_ext --inplace)
+try:
+    import pyximport
+    pyximport.install()
+    import layout
+except: # pragma: no cover
+    pass
+
+from .nilmdb import NilmDB
+from .server import Server
+from .errors import *

nilmdb/server/bulkdata.py

@@ -1,5 +1,7 @@
 # Fixed record size bulk data storage
 
+# Need absolute_import so that "import nilmdb" won't pull in
+# nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
 from __future__ import division
 import nilmdb
@@ -75,7 +77,7 @@ class BulkData(object):
 
         # Get layout, and build format string for struct module
         try:
-            layout = nilmdb.layout.get_named(layout_name)
+            layout = nilmdb.server.layout.get_named(layout_name)
             struct_fmt = '<d'  # Little endian, double timestamp
             struct_mapping = {
                 "int8": 'b',

nilmdb/server/errors.py

@@ -0,0 +1,12 @@
+"""Exceptions"""
+
+class NilmDBError(Exception):
+    """Base exception for NilmDB errors"""
+    def __init__(self, message = "Unspecified error"):
+        Exception.__init__(self, message)
+
+class StreamError(NilmDBError):
+    pass
+
+class OverlapError(NilmDBError):
+    pass

nilmdb/server/layout.pyx

@@ -170,7 +170,7 @@ class Parser(object):
                 if line[0] == '\#':
                     continue
                 (ts, row) = self.layout.parse(line)
-                if ts < last_ts:
+                if ts <= last_ts:
                     raise ValueError("timestamp is not "
                                      "monotonically increasing")
                 last_ts = ts

nilmdb/server/nilmdb.py

@@ -7,11 +7,15 @@ Object that represents a NILM database file.
 Manages both the SQL database and the table storage backend.
 """
 
-# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
+# Need absolute_import so that "import nilmdb" won't pull in
-# but will pull the nilmdb module instead.
+# nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
 import nilmdb
 from nilmdb.utils.printf import *
+from nilmdb.server.interval import (Interval, DBInterval,
+                                    IntervalSet, IntervalError)
+from nilmdb.server import bulkdata
+from nilmdb.server.errors import *
 
 import sqlite3
 import time
@@ -20,12 +24,6 @@ import os
 import errno
 import bisect
 
-import pyximport
-pyximport.install()
-from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
-
-from . import bulkdata
-
 # Note about performance and transactions:
 #
 # Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
@@ -77,17 +75,6 @@ _sql_schema_updates = {
     """,
 }
 
-class NilmDBError(Exception):
-    """Base exception for NilmDB errors"""
-    def __init__(self, message = "Unspecified error"):
-        Exception.__init__(self, message)
-
-class StreamError(NilmDBError):
-    pass
-
-class OverlapError(NilmDBError):
-    pass
-
 @nilmdb.utils.must_close()
 class NilmDB(object):
     verbose = 0

nilmdb/server/server.py

@@ -1,10 +1,11 @@
 """CherryPy-based server for accessing NILM database via HTTP"""
 
-# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
+# Need absolute_import so that "import nilmdb" won't pull in
-# but will pull the nilmdb module instead.
+# nilmdb.py, but will pull the nilmdb module instead.
 from __future__ import absolute_import
-from nilmdb.utils.printf import *
 import nilmdb
+from nilmdb.utils.printf import *
+from nilmdb.server.errors import *
 
 import cherrypy
 import sys
@@ -14,8 +15,6 @@ import simplejson as json
 import decorator
 import traceback
 
-from nilmdb.nilmdb import NilmDBError
-
 try:
     import cherrypy
     cherrypy.tools.json_out
@@ -37,6 +36,14 @@ def chunked_response(func):
     func._cp_config = { 'response.stream': True }
     return func
 
+def response_type(content_type):
+    """Return a decorator-generating function that sets the
+    response type to the specified string."""
+    def wrapper(func, *args, **kwargs):
+        cherrypy.response.headers['Content-Type'] = content_type
+        return func(*args, **kwargs)
+    return decorator.decorator(wrapper)
+
 @decorator.decorator
 def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
     """Decorator to work around CherryPy bug #1200 in a response
@@ -151,7 +158,7 @@ class Stream(NilmApp):
         matching the given keys."""
         try:
             data = self.db.stream_get_metadata(path)
-        except nilmdb.nilmdb.StreamError as e:
+        except nilmdb.server.nilmdb.StreamError as e:
             raise cherrypy.HTTPError("404 Not Found", e.message)
         if key is None:  # If no keys specified, return them all
             key = data.keys()
@@ -214,9 +221,9 @@ class Stream(NilmApp):
 
         # Parse the input data
         try:
-            parser = nilmdb.layout.Parser(layout)
+            parser = nilmdb.server.layout.Parser(layout)
             parser.parse(body)
-        except nilmdb.layout.ParserError as e:
+        except nilmdb.server.layout.ParserError as e:
             raise cherrypy.HTTPError("400 Bad Request",
                                      "error parsing input data: " +
                                      e.message)
@@ -241,7 +248,7 @@ class Stream(NilmApp):
         # Now do the nilmdb insert, passing it the parser full of data.
         try:
             result = self.db.stream_insert(path, start, end, parser.data)
-        except nilmdb.nilmdb.NilmDBError as e:
+        except NilmDBError as e:
             raise cherrypy.HTTPError("400 Bad Request", e.message)
 
         # Done
@@ -272,12 +279,17 @@ class Stream(NilmApp):
     # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
     @cherrypy.expose
     @chunked_response
+    @response_type("text/plain")
     def intervals(self, path, start = None, end = None):
         """
         Get intervals from backend database.  Streams the resulting
         intervals as JSON strings separated by newlines.  This may
         make multiple requests to the nilmdb backend to avoid causing
         it to block for too long.
+
+        Note that the response type is set to 'text/plain' even
+        though we're sending back JSON; this is because we're not
+        really returning a single JSON object.
         """
         if start is not None:
             start = float(start)
@@ -308,6 +320,7 @@ class Stream(NilmApp):
     # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
     @cherrypy.expose
     @chunked_response
+    @response_type("text/plain")
     def extract(self, path, start = None, end = None, count = False):
         """
         Extract data from backend database.  Streams the resulting
@@ -335,7 +348,7 @@ class Stream(NilmApp):
         layout = streams[0][1]
 
         # Get formatter
-        formatter = nilmdb.layout.Formatter(layout)
+        formatter = nilmdb.server.layout.Formatter(layout)
 
         @workaround_cp_bug_1200
         def content(start, end, count):
@@ -390,6 +403,12 @@ class Server(object):
         if self.embedded:
             cherrypy.config.update({ 'environment': 'embedded' })
 
+        # Send a permissive Access-Control-Allow-Origin (CORS) header
+        # with all responses so that browsers can send cross-domain
+        # requests to this server.
+        cherrypy.config.update({ 'response.headers.Access-Control-Allow-Origin':
+                                 '*' })
+
         # Send tracebacks in error responses.  They're hidden by the
         # error_page function for client errors (code 400-499).
         cherrypy.config.update({ 'request.show_tracebacks' : True })
@@ -462,8 +481,10 @@ class Server(object):
         cherrypy.engine.start()
         os._exit = real_exit
 
+        # Signal that the engine has started successfully
         if event is not None:
             event.set()
+
         if blocking:
             try:
                 cherrypy.engine.wait(cherrypy.engine.states.EXITING,

nilmdb/utils/diskusage.py

@@ -1,4 +1,3 @@
-import nilmdb
 import os
 from math import log
 

nilmdb/utils/iteratorizer.py

@@ -1,14 +1,16 @@
 import Queue
 import threading
 import sys
+import contextlib
 
-# This file provides a class that will convert a function that
+# This file provides a context manager that converts a function
-# takes a callback into a generator that returns an iterator.
+# that takes a callback into a generator that returns an iterable.
+# This is done by running the function in a new thread.
 
 # Based partially on http://stackoverflow.com/questions/9968592/
 
 class IteratorizerThread(threading.Thread):
-    def __init__(self, queue, function):
+    def __init__(self, queue, function, curl_hack):
         """
         function: function to execute, which takes the
         callback (provided by this class) as an argument
@@ -17,56 +19,81 @@ class IteratorizerThread(threading.Thread):
         self.function = function
         self.queue = queue
         self.die = False
+        self.curl_hack = curl_hack
 
     def callback(self, data):
-        if self.die:
+        try:
-            raise Exception("should die")
+            if self.die:
-        self.queue.put((1, data))
+                raise Exception() # trigger termination
+            self.queue.put((1, data))
+        except:
+            if self.curl_hack:
+                # We can't raise exceptions, because the pycurl
+                # extension module will unconditionally print the
+                # exception itself, and not pass it up to the caller.
+                # Instead, just return a value that tells curl to
+                # abort.  (-1 would be best, in case we were given 0
+                # bytes, but the extension doesn't support that).
+                self.queue.put((2, sys.exc_info()))
+                return 0
+            raise
 
     def run(self):
         try:
             result = self.function(self.callback)
         except:
-            if sys is not None: # can be None during unclean shutdown
+            self.queue.put((2, sys.exc_info()))
-                self.queue.put((2, sys.exc_info()))
         else:
             self.queue.put((0, result))
 
-class Iteratorizer(object):
+@contextlib.contextmanager
-    def __init__(self, function):
+def Iteratorizer(function, curl_hack = False):
-        """
+    """
-        function: function to execute, which takes the
+    Context manager that takes a function expecting a callback,
-        callback (provided by this class) as an argument
+    and provides an iterable that yields the values passed to that
-        """
+    callback instead.
-        self.function = function
-        self.queue = Queue.Queue(maxsize = 1)
-        self.thread = IteratorizerThread(self.queue, self.function)
-        self.thread.daemon = True
-        self.thread.start()
 
-    def __del__(self):
+    function: function to execute, which takes a callback
-        # If we get garbage collected, try to get rid of the
+    (provided by this context manager) as an argument
-        # thread too by asking it to raise an exception, then
+
-        # draining the queue until it's gone.
+        with iteratorizer(func) as it:
-        self.thread.die = True
+            for i in it:
-        while self.thread.isAlive():
+                print 'callback was passed:', i
+        print 'function returned:', it.retval
+    """
+    queue = Queue.Queue(maxsize = 1)
+    thread = IteratorizerThread(queue, function, curl_hack)
+    thread.daemon = True
+    thread.start()
+
+    class iteratorizer_gen(object):
+        def __init__(self, queue):
+            self.queue = queue
+            self.retval = None
+
+        def __iter__(self):
+            return self
+
+        def next(self):
+            (typ, data) = self.queue.get()
+            if typ == 0:
+                # function has returned
+                self.retval = data
+                raise StopIteration
+            elif typ == 1:
+                # data is available
+                return data
+            else:
+                # callback raised an exception
+                raise data[0], data[1], data[2]
+
+    try:
+        yield iteratorizer_gen(queue)
+    finally:
+        # Ask the thread to die, if it's still running.
+        thread.die = True
+        while thread.isAlive():
             try:
-                self.queue.get(True, 0.01)
+                queue.get(True, 0.01)
-            except: # pragma: no cover
+            except:
                 pass
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        (typ, data) = self.queue.get()
-        if typ == 0:
-            # function returned
-            self.retval = data
-            raise StopIteration
-        elif typ == 1:
-            # data available
-            return data
-        else:
-            # exception
-            raise data[0], data[1], data[2]

nilmdb/utils/timestamper.py

@@ -1,11 +1,10 @@
 """File-like objects that add timestamps to the input lines"""
 
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+from nilmdb.utils import datetime_tz
 
 import time
 import os
-import datetime_tz
 
 class Timestamper(object):
     """A file-like object that adds timestamps to lines of an input file."""

nilmdb/utils/urllib.py

@@ -20,9 +20,6 @@ def urlencode(query):
             v = quote_plus(v)
             l.append(k + '=' + v)
         elif _is_unicode(v):
-            # is there a reasonable way to convert to ASCII?
-            # encode generates a string, but "replace" or "ignore"
-            # lose information and "strict" can raise UnicodeError
             v = quote_plus(v.encode("utf-8","strict"))
             l.append(k + '=' + v)
         else:

setup.cfg

@@ -1,16 +1,23 @@
+[aliases]
+test = nosetests
+
 [nosetests]
-# note: the value doesn't matter, that's why they're empty here
+# Note: values must be set to 1, and have no comments on the same line,
-nocapture=
+# for "python setup.py nosetests" to work correctly.
-nologcapture=        # comment to see cherrypy logs on failure
+nocapture=1
-with-coverage=
+# Comment this out to see CherryPy logs on failure:
-cover-inclusive=
+nologcapture=1
+with-coverage=1
+cover-inclusive=1
 cover-package=nilmdb
-cover-erase=
+cover-erase=1
-##cover-html=         # this works, puts html output in cover/ dir
+# this works, puts html output in cover/ dir:
-##cover-branches=     # need nose 1.1.3 for this
+# cover-html=1
+# need nose 1.1.3 for this:
+# cover-branches=1
 #debug=nose
 #debug-log=nose.log
-stop=
+stop=1
 verbosity=2
 tests=tests
 #tests=tests/test_bulkdata.py
@@ -28,6 +35,6 @@ tests=tests
 #tests=tests/test_iteratorizer.py
 #tests=tests/test_client.py:TestClient.test_client_nilmdb
 #tests=tests/test_nilmdb.py
-#with-profile=
+#with-profile=1
 #profile-sort=time
 ##profile-restrict=10  # doesn't work right, treated as string or something

setup.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+# This is supposed to be using Distribute:
+#
+#   distutils provides a "setup" method.
+#   setuptools is a set of monkeypatches on top of that.
+#   distribute is a particular version/implementation of setuptools.
+#
+# So we don't really know if this is using the old setuptools or the
+# Distribute-provided version of setuptools.
+
+from setuptools import setup, find_packages
+from distutils.extension import Extension
+
+from Cython.Build import cythonize
+
+# Hack to workaround logging/multiprocessing issue:
+# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
+try: import multiprocessing
+except: pass
+
+# Build cython modules.
+cython_modules = cythonize("**/*.pyx")
+
+# Run setup
+setup(name='nilmdb',
+      version = '1.0',
+      url = 'https://git.jim.sh/jim/lees/nilmdb.git',
+      author = 'Jim Paris',
+      author_email = 'jim@jtan.com',
+      tests_require = [ 'nose',
+                        'coverage',
+                        ],
+      setup_requires = [ 'cython',
+                        ],
+      install_requires = [ 'distribute',
+                           'decorator',
+                           ],
+      packages = [ 'nilmdb',
+                   'nilmdb.utils',
+                   'nilmdb.utils.datetime_tz',
+                   'nilmdb.server',
+                   'nilmdb.client',
+                   'nilmdb.cmdline',
+                   ],
+      ext_modules = cython_modules,
+      zip_safe = False,
+      )

tests/data/extract-7

@@ -0,0 +1,124 @@
+# path: /newton/prep
+# layout: PrepData
+# start: 1332496830.0
+# end: 1332496830.999
+1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010

tests/data/prep-20120323T1004-badtimes

@@ -0,0 +1,11 @@
+1332497040.000000 2.56439e+05  2.24775e+05  2.92897e+03  4.66646e+03  7.58491e+03  3.57351e+03  -4.34171e+02  2.98819e+03  
+1332497040.010000 2.51903e+05  2.23202e+05  4.23696e+03  3.49363e+03  8.53493e+03  4.29416e+03  8.49573e+02  2.38189e+03  
+1332497040.020000 2.57625e+05  2.20247e+05  5.47017e+03  1.35872e+03  9.18903e+03  4.56136e+03  2.65599e+03  2.60912e+03  
+1332497040.030000 2.63375e+05  2.20706e+05  4.51842e+03  1.80758e+03  8.17208e+03  4.17463e+03  2.57884e+03  3.32848e+03  
+1332497040.040000 2.59221e+05  2.22346e+05  2.98879e+03  3.66264e+03  6.87274e+03  3.94223e+03  1.25928e+03  3.51786e+03  
+1332497040.050000 2.51918e+05  2.22281e+05  4.22677e+03  2.84764e+03  7.78323e+03  3.81659e+03  8.04944e+02  3.46314e+03  
+1332497040.050000 2.54478e+05  2.21701e+05  5.61366e+03  1.02262e+03  9.26581e+03  3.50152e+03  1.29331e+03  3.07271e+03  
+1332497040.060000 2.59568e+05  2.22945e+05  4.97190e+03  1.28250e+03  8.62081e+03  4.06316e+03  1.85717e+03  2.61990e+03  
+1332497040.070000 2.57269e+05  2.23697e+05  3.60527e+03  3.05749e+03  7.22363e+03  4.90330e+03  1.93736e+03  2.35357e+03  
+1332497040.080000 2.52274e+05  2.21438e+05  5.01228e+03  2.86309e+03  7.87115e+03  4.80448e+03  2.18291e+03  2.93397e+03  
+1332497040.090000 2.56468e+05  2.19205e+05  6.29804e+03  8.09467e+02  9.12895e+03  3.52055e+03  2.16980e+03  3.88739e+03  

tests/test_bulkdata.py

@@ -2,8 +2,6 @@
 
 import nilmdb
 from nilmdb.utils.printf import *
-import nilmdb.bulkdata
-
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
@@ -12,7 +10,8 @@ from testutil.helpers import *
 
 testdb = "tests/bulkdata-testdb"
 
-from nilmdb.bulkdata import BulkData
+import nilmdb.server.bulkdata
+from nilmdb.server.bulkdata import BulkData
 
 class TestBulkData(object):
 

tests/test_client.py

@@ -2,9 +2,9 @@
 
 import nilmdb
 from nilmdb.utils.printf import *
+from nilmdb.utils import timestamper
 from nilmdb.client import ClientError, ServerError
-
+from nilmdb.utils import datetime_tz
-import datetime_tz
 
 from nose.tools import *
 from nose.tools import assert_raises
@@ -158,13 +158,13 @@ class TestClient(object):
         rate = 120
 
         # First try a nonexistent path
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/no-such-path", data)
         in_("404 Not Found", str(e.exception))
 
         # Now try reversed timestamps
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         data = reversed(list(data))
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
@@ -173,7 +173,7 @@ class TestClient(object):
 
         # Now try empty data (no server request made)
         empty = cStringIO.StringIO("")
-        data = nilmdb.timestamper.TimestamperRate(empty, start, 120)
+        data = timestamper.TimestamperRate(empty, start, 120)
         result = client.stream_insert("/newton/prep", data)
         eq_(result, None)
 
@@ -185,7 +185,7 @@ class TestClient(object):
         in_("no data provided", str(e.exception))
 
         # Specify start/end (starts too late)
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
                                           start + 5, start + 120)
@@ -194,7 +194,7 @@ class TestClient(object):
             str(e.exception))
 
         # Specify start/end (ends too early)
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
                                           start, start + 1)
@@ -205,7 +205,7 @@ class TestClient(object):
             str(e.exception))
 
         # Now do the real load
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         result = client.stream_insert("/newton/prep", data,
                                       start, start + 119.999777)
         eq_(result, "ok")
@@ -216,7 +216,7 @@ class TestClient(object):
         eq_(intervals, [[start, start + 119.999777]])
 
         # Try some overlapping data -- just insert it again
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
         in_("400 Bad Request", str(e.exception))
@@ -227,7 +227,7 @@ class TestClient(object):
         client = nilmdb.Client(url = "http://localhost:12380/")
 
         for x in client.stream_extract("/newton/prep", 123, 123):
-            raise Exception("shouldn't be any data for this request")
+            raise AssertionError("shouldn't be any data for this request")
 
         with assert_raises(ClientError) as e:
             client.stream_remove("/newton/prep", 123, 120)
@@ -281,28 +281,44 @@ class TestClient(object):
             in_("404 Not Found", str(e.exception))
             in_("No such stream", str(e.exception))
 
-    def test_client_7_chunked(self):
+    def test_client_7_headers(self):
         # Make sure that /stream/intervals and /stream/extract
-        # properly return streaming, chunked response.  Pokes around
+        # properly return streaming, chunked, text/plain response.
-        # in client.http internals a bit to look at the response
+        # Pokes around in client.http internals a bit to look at the
-        # headers.
+        # response headers.
 
         client = nilmdb.Client(url = "http://localhost:12380/")
+        http = client.http
 
         # Use a warning rather than returning a test failure, so that we can
         # still disable chunked responses for debugging.
-        x = client.http.get("stream/intervals", { "path": "/newton/prep" },
+
+        # Intervals
+        x = http.get("stream/intervals", { "path": "/newton/prep" },
                             retjson=False)
         lines_(x, 1)
-        if "transfer-encoding: chunked" not in client.http._headers.lower():
+        if "Transfer-Encoding: chunked" not in http._headers:
             warnings.warn("Non-chunked HTTP response for /stream/intervals")
+        if "Content-Type: text/plain;charset=utf-8" not in http._headers:
+            raise AssertionError("/stream/intervals is not text/plain:\n" +
+                                 http._headers)
 
-        x = client.http.get("stream/extract",
+        # Extract
+        x = http.get("stream/extract",
                             { "path": "/newton/prep",
                               "start": "123",
                               "end": "123" }, retjson=False)
-        if "transfer-encoding: chunked" not in client.http._headers.lower():
+        if "Transfer-Encoding: chunked" not in http._headers:
             warnings.warn("Non-chunked HTTP response for /stream/extract")
+        if "Content-Type: text/plain;charset=utf-8" not in http._headers:
+            raise AssertionError("/stream/extract is not text/plain:\n" +
+                                 http._headers)
+
+        # Make sure Access-Control-Allow-Origin gets set
+        if "Access-Control-Allow-Origin: " not in http._headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in /stream/extract response:\n" +
+                                 http._headers)
 
     def test_client_8_unicode(self):
         # Basic Unicode tests

tests/test_cmdline.py

@@ -3,12 +3,12 @@
 import nilmdb
 from nilmdb.utils.printf import *
 import nilmdb.cmdline
+from nilmdb.utils import datetime_tz
 
 import unittest
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
-import datetime_tz
 import os
 import re
 import shutil
@@ -128,8 +128,8 @@ class TestCmdline(object):
         with open(file) as f:
             contents = f.read()
             if contents != self.captured:
-                #print contents[1:1000] + "\n"
+                print contents[1:1000] + "\n"
-                #print self.captured[1:1000] + "\n"
+                print self.captured[1:1000] + "\n"
                 raise AssertionError("captured data doesn't match " + file)
 
     def matchfilecount(self, file):
@@ -340,9 +340,10 @@ class TestCmdline(object):
         eq_(cmd.parse_time("hi there 20120405 1400-0400 testing! 123"), test)
         eq_(cmd.parse_time("20120405 1800 UTC"), test)
         eq_(cmd.parse_time("20120405 1400-0400 UTC"), test)
-        for badtime in [ "20120405 1400-9999", "hello", "-", "", "14:00" ]:
+        for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
             with assert_raises(ValueError):
                 x = cmd.parse_time(badtime)
+        x = cmd.parse_time("now")
         eq_(cmd.parse_time("snapshot-20120405-140000.raw.gz"), test)
         eq_(cmd.parse_time("prep-20120405T1400"), test)
 
@@ -369,6 +370,14 @@ class TestCmdline(object):
         with open("tests/data/prep-20120323T1004-timestamped") as input:
             self.ok("insert --none /newton/prep", input)
 
+        # insert pre-timestamped data, with bad times (non-monotonic)
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-badtimes") as input:
+            self.fail("insert --none /newton/prep", input)
+            self.contain("error parsing input data")
+            self.contain("line 7:")
+            self.contain("timestamp is not monotonically increasing")
+
         # insert data with normal timestamper from filename
         os.environ['TZ'] = "UTC"
         self.ok("insert --rate 120 /newton/prep "
@@ -440,6 +449,17 @@ class TestCmdline(object):
         self.ok("list --detail")
         lines_(self.captured, 8)
 
+        # Verify the "raw timestamp" output
+        self.ok("list --detail --path *prep --timestamp-raw "
+                "--start='23 Mar 2012 10:05:15.50'")
+        lines_(self.captured, 2)
+        self.contain("[ 1332497115.5 -> 1332497159.991668 ]")
+
+        self.ok("list --detail --path *prep -T "
+                "--start='23 Mar 2012 10:05:15.612'")
+        lines_(self.captured, 2)
+        self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
+
     def test_08_extract(self):
         # nonexistent stream
         self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")
@@ -495,6 +515,8 @@ class TestCmdline(object):
         test(4, "10:00:30.008333", "10:00:30.025")
         test(5, "10:00:30", "10:00:31", extra="--annotate --bare")
         test(6, "10:00:30", "10:00:31", extra="-b")
+        test(7, "10:00:30", "10:00:30.999", extra="-a -T")
+        test(7, "10:00:30", "10:00:30.999", extra="-a --timestamp-raw")
 
         # all data put in by tests
         self.ok("extract -a /newton/prep --start 2000-01-01 --end 2020-01-01")

tests/test_interval.py

@@ -2,13 +2,14 @@
 
 import nilmdb
 from nilmdb.utils.printf import *
-import datetime_tz
+from nilmdb.utils import datetime_tz
 
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
 
-from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
+from nilmdb.server.interval import (Interval, DBInterval,
+                                    IntervalSet, IntervalError)
 
 from testutil.helpers import *
 import unittest

tests/test_iteratorizer.py

@@ -13,6 +13,7 @@ def func_with_callback(a, b, callback):
     callback(a)
     callback(b)
     callback(a+b)
+    return "return value"
 
 class TestIteratorizer(object):
     def test(self):
@@ -25,22 +26,21 @@ class TestIteratorizer(object):
         eq_(self.result, "123")
 
         # Now make it an iterator
-        it = nilmdb.utils.Iteratorizer(
-            lambda x:
-            func_with_callback(1, 2, x))
         result = ""
-        for i in it:
+        f = lambda x: func_with_callback(1, 2, x)
-            result += str(i)
+        with nilmdb.utils.Iteratorizer(f) as it:
-        eq_(result, "123")
-
-        # Make sure things work when an exception occurs
-        it = nilmdb.utils.Iteratorizer(
-            lambda x:
-            func_with_callback(1, "a", x))
-        result = ""
-        with assert_raises(TypeError) as e:
             for i in it:
                 result += str(i)
+        eq_(result, "123")
+        eq_(it.retval, "return value")
+
+        # Make sure things work when an exception occurs
+        result = ""
+        with nilmdb.utils.Iteratorizer(
+            lambda x: func_with_callback(1, "a", x)) as it:
+            with assert_raises(TypeError) as e:
+                for i in it:
+                    result += str(i)
         eq_(result, "1a")
 
         # Now try to trigger the case where we stop iterating
@@ -48,8 +48,14 @@ class TestIteratorizer(object):
         # itself.  This doesn't have a particular result in the test,
         # but gains coverage.
         def foo():
-            it = nilmdb.utils.Iteratorizer(
+            with nilmdb.utils.Iteratorizer(f) as it:
-                lambda x:
+                it.next()
-                func_with_callback(1, 2, x))
-            it.next()
         foo()
+        eq_(it.retval, None)
+
+        # Do the same thing when the curl hack is applied
+        def foo():
+            with nilmdb.utils.Iteratorizer(f, curl_hack = True) as it:
+                it.next()
+        foo()
+        eq_(it.retval, None)

tests/test_layout.py

@@ -22,17 +22,17 @@ import unittest
 
 from testutil.helpers import *
 
-from nilmdb.layout import *
+from nilmdb.server.layout import *
 
 class TestLayouts(object):
     # Some nilmdb.layout tests.  Not complete, just fills in missing
     # coverage.
     def test_layouts(self):
-        x = nilmdb.layout.get_named("PrepData")
+        x = nilmdb.server.layout.get_named("PrepData")
-        y = nilmdb.layout.get_named("float32_8")
+        y = nilmdb.server.layout.get_named("float32_8")
         eq_(x.count, y.count)
         eq_(x.datatype, y.datatype)
-        y = nilmdb.layout.get_named("float32_7")
+        y = nilmdb.server.layout.get_named("float32_7")
         ne_(x.count, y.count)
         eq_(x.datatype, y.datatype)
 
@@ -89,11 +89,23 @@ class TestLayouts(object):
         # non-monotonic
         parser = Parser(name_raw)
         data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.000000 1 2 3 4 5 6\n" )
+                 "1234567890.099999 1 2 3 4 5 6\n" )
         with assert_raises(ParserError) as e:
             parser.parse(data)
         in_("not monotonically increasing", str(e.exception))
 
+        parser = Parser(name_raw)
+        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
+                 "1234567890.100000 1 2 3 4 5 6\n" )
+        with assert_raises(ParserError) as e:
+            parser.parse(data)
+        in_("not monotonically increasing", str(e.exception))
+
+        parser = Parser(name_raw)
+        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
+                 "1234567890.100001 1 2 3 4 5 6\n" )
+        parser.parse(data)
+
         # RawData with values out of bounds
         parser = Parser(name_raw)
         data = ( "1234567890.000000 1 2 3 4 500000 6\n" +

tests/test_nilmdb.py

@@ -113,7 +113,8 @@ class TestBlockingServer(object):
             self.server.start(blocking = True, event = event)
         thread = threading.Thread(target = run_server)
         thread.start()
-        event.wait(timeout = 2)
+        if not event.wait(timeout = 10):
+            raise AssertionError("server didn't start in 10 seconds")
 
         # Send request to exit.
         req = urlopen("http://127.0.0.1:12380/exit/", timeout = 1)

tests/test_rbtree.py

@@ -6,7 +6,7 @@ from nilmdb.utils.printf import *
 from nose.tools import *
 from nose.tools import assert_raises
 
-from nilmdb.rbtree import RBTree, RBNode
+from nilmdb.server.rbtree import RBTree, RBNode
 
 from testutil.helpers import *
 import unittest

tests/test_timestamper.py

@@ -1,7 +1,6 @@
 import nilmdb
 from nilmdb.utils.printf import *
-
+from nilmdb.utils import datetime_tz
-import datetime_tz
 
 from nose.tools import *
 from nose.tools import assert_raises
@@ -11,6 +10,8 @@ import cStringIO
 
 from testutil.helpers import *
 
+from nilmdb.utils import timestamper
+
 class TestTimestamper(object):
 
     # Not a very comprehensive test, but it's good enough.
@@ -27,20 +28,20 @@ class TestTimestamper(object):
 
         # full
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
+        ts = timestamper.TimestamperRate(input, start, 8000)
         foo = ts.readlines()
         eq_(foo, join(lines_out))
         in_("TimestamperRate(..., start=", str(ts))
 
         # first 30 or so bytes means the first 2 lines
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
+        ts = timestamper.TimestamperRate(input, start, 8000)
         foo = ts.readlines(30)
         eq_(foo, join(lines_out[0:2]))
 
         # stop iteration early
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
+        ts = timestamper.TimestamperRate(input, start, 8000,
                                                 1332561600.000200)
         foo = ""
         for line in ts:
@@ -49,21 +50,21 @@ class TestTimestamper(object):
 
         # stop iteration early (readlines)
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
+        ts = timestamper.TimestamperRate(input, start, 8000,
                                                 1332561600.000200)
         foo = ts.readlines()
         eq_(foo, join(lines_out[0:2]))
 
         # stop iteration really early
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
+        ts = timestamper.TimestamperRate(input, start, 8000,
                                                 1332561600.000000)
         foo = ts.readlines()
         eq_(foo, "")
 
         # use iterator
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
+        ts = timestamper.TimestamperRate(input, start, 8000)
         foo = ""
         for line in ts:
             foo += line
@@ -71,21 +72,21 @@ class TestTimestamper(object):
 
         # check that TimestamperNow gives similar result
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperNow(input)
+        ts = timestamper.TimestamperNow(input)
         foo = ts.readlines()
         ne_(foo, join(lines_out))
         eq_(len(foo), len(join(lines_out)))
         eq_(str(ts), "TimestamperNow(...)")
 
         # Test passing a file (should be empty)
-        ts = nilmdb.timestamper.TimestamperNow("/dev/null")
+        ts = timestamper.TimestamperNow("/dev/null")
         for line in ts:
             raise AssertionError
         ts.close()
 
         # Test the null timestamper
         input = cStringIO.StringIO(join(lines_out))  # note: lines_out
-        ts = nilmdb.timestamper.TimestamperNull(input)
+        ts = timestamper.TimestamperNull(input)
         foo = ts.readlines()
         eq_(foo, join(lines_out))
         eq_(str(ts), "TimestamperNull(...)")

time-bxintersect

@@ -1,54 +0,0 @@
-nosetests
-
-32: 386 μs (12.0625 μs each)
-64: 672.102 μs (10.5016 μs each)
-128: 1510.86 μs (11.8036 μs each)
-256: 2782.11 μs (10.8676 μs each)
-512: 5591.87 μs (10.9216 μs each)
-1024: 12812.1 μs (12.5119 μs each)
-2048: 21835.1 μs (10.6617 μs each)
-4096: 46059.1 μs (11.2449 μs each)
-8192: 114127 μs (13.9315 μs each)
-16384: 181217 μs (11.0606 μs each)
-32768: 419649 μs (12.8067 μs each)
-65536: 804320 μs (12.2729 μs each)
-131072: 1.73534e+06 μs (13.2396 μs each)
-262144: 3.74451e+06 μs (14.2842 μs each)
-524288: 8.8694e+06 μs (16.917 μs each)
-1048576: 1.69993e+07 μs (16.2118 μs each)
-2097152: 3.29387e+07 μs (15.7064 μs each)
-   |                                                                            
-   +3.29387e+07                                                             *   
-   |                                                                    ----    
-   |                                                               -----        
-   |                                                           ----             
-   |                                                      -----                 
-   |                                                 -----                      
-   |                                             ----                           
-   |                                        -----                               
-   |                                   -----                                    
-   |                               ----                                         
-   |                          -----                                             
-   |                      ----                                                  
-   |                 -----                                                      
-   |              ---                                                           
-   |           ---                                                              
-   |        ---                                                                 
-   | -------                                                                    
----+386---------------------------------------------------------------------+---
-   +32                                                          +2.09715e+06    
-
-name                                 #n            tsub      ttot      tavg
-..vl/lees/bucket/nilm/nilmdb/nilmdb/interval.py.__iadd__:184  4194272          10.025323     30.262723     0.000007    
-..evl/lees/bucket/nilm/nilmdb/nilmdb/interval.py.__init__:27  4194272          24.715377     24.715377     0.000006    
-../lees/bucket/nilm/nilmdb/nilmdb/interval.py.intersects:239  4194272          6.705053      12.577620     0.000003    
-..im/devl/lees/bucket/nilm/nilmdb/tests/aplotter.py.plot:404  1                0.000048      0.001412      0.001412    
-../lees/bucket/nilm/nilmdb/tests/aplotter.py.plot_double:311  1                0.000106      0.001346      0.001346    
-..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.plot_data:201  1                0.000098      0.000672      0.000672    
-..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.plot_line:241  16               0.000298      0.000496      0.000031    
-..jim/devl/lees/bucket/nilm/nilmdb/nilmdb/printf.py.printf:4  17               0.000252      0.000334      0.000020    
-..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.transposed:39  1                0.000229      0.000235      0.000235    
-..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.y_reversed:45  1                0.000151      0.000174      0.000174    
-
-name           tid              fname                      ttot      scnt
-_MainThread           47269783682784        ..b/python2.7/threading.py.setprofile:88  64.746000     1              

timeit.sh

@@ -1,22 +0,0 @@
-./nilmtool.py destroy /bpnilm/2/raw
-./nilmtool.py create /bpnilm/2/raw RawData
-
-if false; then
-    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 -r 8000 /bpnilm/2/raw
-    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 -r 8000 /bpnilm/2/raw
-else
-    # 170 hours, about 98 gigs uncompressed:
-    for i in $(seq 2000 2016); do
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-010001 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-020002 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-030003 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-040004 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-050005 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-060006 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-070007 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-080008 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-090009 -r 8000 /bpnilm/2/raw
-	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-100010 -r 8000 /bpnilm/2/raw
-    done
-fi
-