Set up default SIGPIPE handler

This lets you do something like "nilmtool extract | head" without
triggering backtraces.

.coveragerc

@@ -7,4 +7,4 @@
 exclude_lines =
 	pragma: no cover
 	if 0:
-omit = nilmdb/utils/datetime_tz*
+omit = nilmdb/utils/datetime_tz*,nilmdb/scripts,nilmdb/_version.py

.gitattributes

@@ -0,0 +1 @@
+nilmdb/_version.py export-subst

.gitignore

@@ -8,16 +8,18 @@ docs/*.html
 build/
 *.pyc
 nilmdb/server/interval.c
-nilmdb/server/interval.so
 nilmdb/server/layout.c
-nilmdb/server/layout.so
 nilmdb/server/rbtree.c
-nilmdb/server/rbtree.so
+*.so
 
 # Setup junk
 dist/
 nilmdb.egg-info/
 
+# This gets generated as needed by setup.py
+MANIFEST.in
+MANIFEST
+
 # Misc
 timeit*out
 

.pylintrc

@@ -0,0 +1,250 @@
+# -*- conf -*-
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=datetime_tz
+
+# Pickle collected data for later comparisons.
+persistent=no
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+disable=C0111,R0903,R0201,R0914,R0912,W0142,W0703,W0702
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html
+output-format=parseable
+
+# Include message's id in output
+include-ids=yes
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (RP0004).
+comment=no
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, add a predefined set of Zope acquired attributes
+# to generated-members.
+zope=no
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E0201 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=80
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the beginning of the name of dummy variables
+# (i.e. not used).
+dummy-variables-rgx=_|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=apply,input
+
+# Regular expression which should only match correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression which should only match correct module level names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|version)$
+
+# Regular expression which should only match correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression which should only match correct function names
+function-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct method names
+method-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct argument names
+argument-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct variable names
+variable-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Regular expression which should only match functions or classes name which do
+# not require a docstring
+no-docstring-rgx=__.*__
+
+
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defines in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branchs=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception

Makefile

@@ -1,12 +1,46 @@
+# By default, run the tests.
 all: test
 
+version:
+	python setup.py version
+
+build:
+	python setup.py build_ext --inplace
+
+dist: sdist
+sdist:
+	python setup.py sdist
+
+install:
+	python setup.py install
+
+develop:
+	python setup.py develop
+
+docs:
+	make -C docs
+
 lint:
-	pylint -f parseable nilmdb
+	pylint --rcfile=.pylintrc nilmdb
 
 test:
-	python runtests.py
+ifeq ($(INSIDE_EMACS), t)
+# Use the slightly more flexible script
+	python setup.py build_ext --inplace
+	python tests/runtests.py
+else
+# Let setup.py check dependencies, build stuff, and run the test
+	python setup.py nosetests
+endif
 
 clean::
 	find . -name '*pyc' | xargs rm -f
 	rm -f .coverage
 	rm -rf tests/*testdb*
+	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so MANIFEST.in
+	make -C docs clean
+
+gitclean::
+	git clean -dXf
+
+.PHONY: all version build dist sdist install docs lint test clean gitclean

README.txt

@@ -3,8 +3,29 @@ by Jim Paris <jim@jtan.com>
 
 Prerequisites:
 
-  sudo apt-get install python2.7 python-cherrypy3 python-decorator python-nose python-coverage python-setuptools
+  # Runtime and build environments
+  sudo apt-get install python2.7 python2.7-dev python-setuptools cython
+
+  # Base NilmDB dependencies
+  sudo apt-get install python-cherrypy3 python-decorator python-simplejson
+  sudo apt-get install python-requests python-dateutil python-tz python-psutil
+
+  # Other dependencies (required by some modules)
+  sudo apt-get install python-numpy
+
+  # Tools for running tests
+  sudo apt-get install python-nose python-coverage
+
+Test:
+  python setup.py nosetests
 
 Install:
 
   python setup.py install
+
+Usage:
+
+  nilmdb-server --help
+  nilmtool --help
+
+See docs/wsgi.md for info on setting up a WSGI application in Apache.

docs/design.md

@@ -140,7 +140,7 @@ Speed
 
 - Next slowdown target is nilmdb.layout.Parser.parse().
     - Rewrote parsers using cython and sscanf
-    - Stats (rev 10831), with _add_interval disabled
+    - Stats (rev 10831), with `_add_interval` disabled
 
         layout.pyx.Parser.parse:128        6303 sec, 262k calls
          layout.pyx.parse:63               13913 sec, 5.1g calls
@@ -186,6 +186,19 @@ IntervalSet speed
     - rbtree and interval converted to cython:
       8.4 μS, total 12 s, 134 MB RAM
 
+- Would like to move Interval itself back to Python so other
+  non-cythonized code like client code can use it more easily.
+  Testing speed with just `test_interval` being tested, with
+  `range(5,22)`, using `/usr/bin/time -v python tests/runtests.py`,
+  times recorded for 2097152:
+    - 52ae397 (Interval in cython):
+	  12.6133 μs each, ratio 0.866533, total 47 sec, 399 MB RAM
+	- 9759dcf (Interval in python):
+	  21.2937 μs each, ratio 1.462870, total 83 sec, 1107 MB RAM
+  That's a huge difference!  Instead, will keep Interval and DBInterval
+  cythonized inside nilmdb, and just have an additional copy in
+  nilmdb.utils for clients to use.
+
 Layouts
 -------
 Current/old design has specific layouts: RawData, PrepData, RawNotchedData.
@@ -266,3 +279,145 @@ Each table contains:
   from the end of the file will not shorten it; it will only be
   deleted when it has been fully filled and all of the data has been
   subsequently removed.
+
+
+Rocket
+------
+
+Original design had the nilmdb.nilmdb thread (through bulkdata)
+convert from on-disk layout to a Python list, and then the
+nilmdb.server thread (from cherrypy) converts to ASCII.  For at least
+the extraction side of things, it's easy to pass the bulkdata a layout
+name instead, and have it convert directly from on-disk to ASCII
+format, because this conversion can then be shoved into a C module.
+This module, which provides a means for converting directly from
+on-disk format to ASCII or Python lists, is the "rocket" interface.
+Python is still used to manage the files and figure out where the
+data should go; rocket just puts binary data directly in or out of
+those files at specified locations.
+
+Before rocket, testing speed with uint16_6 data, with an end-to-end
+test (extracting data with nilmtool):
+
+- insert: 65 klines/sec
+- extract: 120 klines/sec
+
+After switching to the rocket design, but using the Python version
+(pyrocket):
+
+- insert: 57 klines/sec
+- extract: 120 klines/sec
+
+After switching to a C extension module (rocket.c)
+
+- insert: 74 klines/sec through insert.py; 99.6 klines/sec through nilmtool
+- extract: 335 klines/sec
+
+After client block updates (described below):
+
+- insert: 180 klines/sec through nilmtool (pre-timestamped)
+- extract: 390 klines/sec through nilmtool
+
+Using "insert --timestamp" or "extract --bare" cuts the speed in half.
+
+Blocks versus lines
+-------------------
+
+Generally want to avoid parsing the bulk of the data as lines if
+possible, and transfer things in bigger blocks at once.
+
+Current places where we use lines:
+
+- All data returned by `client.stream_extract`, since it comes from
+  `httpclient.get_gen`, which iterates over lines.  Not sure if this
+  should be changed, because a `nilmtool extract` is just about the
+  same speed as `curl -q .../stream/extract`!
+
+- `client.StreamInserter.insert_iter` and
+  `client.StreamInserter.insert_line`, which should probably get
+  replaced with block versions.  There's no real need to keep
+  updating the timestamp every time we get a new line of data.
+
+  - Finished.  Just a single insert() that takes any length string and
+    does very little processing until it's time to send it to the
+	server.
+
+Timestamps
+----------
+
+Timestamps are currently double-precision floats (64 bit).  Since the
+mantissa is 53-bit, this can only represent about 15-17 significant
+figures, and microsecond Unix timestamps like 1222333444.000111 are
+already 16 significant figures.  Rounding is therefore an issue;
+it's hard to sure that converting from ASCII, then back to ASCII,
+will always give the same result.
+
+Also, if the client provides a floating point value like 1.9999999999,
+we need to be careful that we don't store it as 1.9999999999 but later
+print it as 2.000000, because then round-trips change the data.
+
+Possible solutions:
+
+- When the client provides a floating point value to the server,
+  always round to the 6th decimal digit before verifying & storing.
+  Good for compatibility and simplicity.  But still might have rounding
+  issues, and clients will also need to round when doing their own
+  verification.  Having every piece of code need to know which digit
+  to round at is not ideal.
+
+- Always store int64 timestamps on the server, representing
+  microseconds since epoch.  int64 timestamps are used in all HTTP
+  parameters, in insert/extract ASCII strings, client API, commandline
+  raw timestamps, etc.  Pretty big change.
+
+  This is what we'll go with...
+
+  - Client programs that interpret the timestamps as doubles instead
+    of ints will remain accurate until 2^53 microseconds, or year
+    2255.
+
+  - On insert, maybe it's OK to send floating point microsecond values
+    (1234567890123456.0), just to cope with clients that want to print
+    everything as a double.  Server could try parsing as int64, and if
+    that fails, parse as double and truncate to int64.  However, this
+    wouldn't catch imprecise inputs like "1.23456789012e+15".  But
+    maybe that can just be ignored; it's likely to cause a
+    non-monotonic error at the client.
+
+  - Timestamps like 1234567890.123456 never show up anywhere, except
+    for interfacing to datetime_tz etc.  Command line "raw timestamps"
+    are always printed as int64 values, and a new format
+    "@1234567890123456" is added to the parser for specifying them
+    exactly.
+
+Binary interface
+----------------
+
+The ASCII interface is too slow for high-bandwidth processing, like
+sinefits, prep, etc.  A binary interface was added so that you can
+extract the raw binary out of the bulkdata storage.  This binary is
+a little-endian format, e.g. in C a uint16_6 stream would be:
+
+    #include <endian.h>
+    #include <stdint.h>
+    struct {
+        int64_t timestamp_le;
+        uint16_t data_le[6];
+    } __attribute__((packed));
+
+Remember to byteswap (with e.g. `letoh` in C)!
+
+This interface is used by the new `nilmdb.client.numpyclient.NumpyClient`
+class, which is a subclass of the normal `nilmcb.client.client.Client`
+and has all of the same functions.  It adds three new functions:
+
+- `stream_extract_numpy` to extract data as a Numpy array
+
+- `stream_insert_numpy` to insert data as a Numpy array
+
+- `stream_insert_numpy_context` is the context manager for
+  incrementally inserting data
+
+It is significantly faster!  It is about 20 times faster to decimate a
+stream with `nilm-decimate` when the filter code is using the new
+binary/numpy interface.

docs/wsgi.md

@@ -0,0 +1,32 @@
+WSGI Application in Apache
+--------------------------
+
+Install `apache2` and `libapache2-mod-wsgi`
+
+We'll set up the database server at URL `http://myhost.com/nilmdb`.
+The database will be stored in `/home/nilm/db`, and the process will
+run as user `nilm`, group `nilm`.
+
+First, create a WSGI script `/home/nilm/nilmdb.wsgi` containing:
+
+    import nilmdb.server
+    application = nilmdb.server.wsgi_application("/home/nilm/db", "/nilmdb")
+
+The first parameter is the local filesystem path, and the second
+parameter is the path part of the URL.
+
+Then, set up Apache with a configuration like:
+
+    <VirtualHost>
+        WSGIScriptAlias /nilmdb /home/nilm/nilmdb.wsgi
+        WSGIApplicationGroup nilmdb-appgroup
+        WSGIProcessGroup nilmdb-procgroup
+        WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilm group=nilm
+
+        # Access control example:
+        <Location /nilmdb>
+            Order deny,allow
+            Deny from all
+            Allow from 1.2.3.4
+        </Location>
+    </VirtualHost>

extras/nilmtool-bash-completion.sh

@@ -0,0 +1,20 @@
+# To enable bash completion:
+#
+# 1. Ensure python-argcomplete is installed:
+#       pip install argcomplete
+# 2. Source this file:
+#       . nilmtool-bash-completion.sh
+
+_nilmtool_argcomplete() {
+    local IFS=$(printf "\013")
+    COMPREPLY=( $(IFS="$IFS" \
+                  COMP_LINE="$COMP_LINE" \
+	          COMP_WORDBREAKS="$COMP_WORDBREAKS" \
+                  COMP_POINT="$COMP_POINT" \
+                  _ARGCOMPLETE=1 \
+                  "$1" 8>&1 9>&2 1>/dev/null 2>/dev/null) )
+    if [[ $? != 0 ]]; then
+        unset COMPREPLY
+    fi
+}
+complete -o nospace -F _nilmtool_argcomplete nilmtool

nilmdb/__init__.py

@@ -1,4 +1,10 @@
 """Main NilmDB import"""
 
-from server import NilmDB, Server
+# These aren't imported automatically, because loading the server
-from client import Client
+# stuff isn't always necessary.
+#from nilmdb.server import NilmDB, Server
+#from nilmdb.client import Client
+
+from nilmdb._version import get_versions
+__version__ = get_versions()['version']
+del get_versions

nilmdb/_version.py

@@ -0,0 +1,197 @@
+
+IN_LONG_VERSION_PY = True
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by githubs download-from-tag
+# feature). Distribution tarballs (build by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
+
+# This file is released into the public domain. Generated by
+# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+
+# these strings will be replaced by git during git-archive
+git_refnames = "$Format:%d$"
+git_full = "$Format:%H$"
+
+
+import subprocess
+import sys
+
+def run_command(args, cwd=None, verbose=False):
+    try:
+        # remember shell=False, so use git.cmd on windows, not just git
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
+    except EnvironmentError:
+        e = sys.exc_info()[1]
+        if verbose:
+            print("unable to run %s" % args[0])
+            print(e)
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version >= '3':
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % args[0])
+        return None
+    return stdout
+
+
+import sys
+import re
+import os.path
+
+def get_expanded_variables(versionfile_source):
+    # the code embedded in _version.py can just fetch the value of these
+    # variables. When used from setup.py, we don't want to import
+    # _version.py, so we do it with a regexp instead. This function is not
+    # used from _version.py.
+    variables = {}
+    try:
+        for line in open(versionfile_source,"r").readlines():
+            if line.strip().startswith("git_refnames ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["refnames"] = mo.group(1)
+            if line.strip().startswith("git_full ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["full"] = mo.group(1)
+    except EnvironmentError:
+        pass
+    return variables
+
+def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
+    refnames = variables["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("variables are unexpanded, not using")
+        return {} # unexpanded, so not in an unpacked git-archive tarball
+    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    for ref in list(refs):
+        if not re.search(r'\d', ref):
+            if verbose:
+                print("discarding '%s', no digits" % ref)
+            refs.discard(ref)
+            # Assume all version tags have a digit. git's %d expansion
+            # behaves like git log --decorate=short and strips out the
+            # refs/heads/ and refs/tags/ prefixes that would let us
+            # distinguish between branches and tags. By ignoring refnames
+            # without digits, we filter out many common branch names like
+            # "release" and "stabilization", as well as "HEAD" and "master".
+    if verbose:
+        print("remaining refs: %s" % ",".join(sorted(refs)))
+    for ref in sorted(refs):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix):]
+            if verbose:
+                print("picking %s" % r)
+            return { "version": r,
+                     "full": variables["full"].strip() }
+    # no suitable tags, so we use the full revision id
+    if verbose:
+        print("no suitable tags, using full revision id")
+    return { "version": variables["full"].strip(),
+             "full": variables["full"].strip() }
+
+def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
+    # this runs 'git' from the root of the source tree. That either means
+    # someone ran a setup.py command (and this code is in versioneer.py, so
+    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
+    # the source tree), or someone ran a project-specific entry point (and
+    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
+    # containing directory is somewhere deeper in the source tree). This only
+    # gets called if the git-archive 'subst' variables were *not* expanded,
+    # and _version.py hasn't already been rewritten with a short version
+    # string, meaning we're inside a checked out source tree.
+
+    try:
+        here = os.path.abspath(__file__)
+    except NameError:
+        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
+        return {} # not always correct
+
+    # versionfile_source is the relative path from the top of the source tree
+    # (where the .git directory might live) to this file. Invert this to find
+    # the root from __file__.
+    root = here
+    if IN_LONG_VERSION_PY:
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        root = os.path.dirname(here)
+    if not os.path.exists(os.path.join(root, ".git")):
+        if verbose:
+            print("no .git in %s" % root)
+        return {}
+
+    GIT = "git"
+    if sys.platform == "win32":
+        GIT = "git.cmd"
+    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
+                         cwd=root)
+    if stdout is None:
+        return {}
+    if not stdout.startswith(tag_prefix):
+        if verbose:
+            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
+        return {}
+    tag = stdout[len(tag_prefix):]
+    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
+    if stdout is None:
+        return {}
+    full = stdout.strip()
+    if tag.endswith("-dirty"):
+        full += "-dirty"
+    return {"version": tag, "full": full}
+
+
+def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
+    if IN_LONG_VERSION_PY:
+        # We're running from _version.py. If it's from a source tree
+        # (execute-in-place), we can work upwards to find the root of the
+        # tree, and then check the parent directory for a version string. If
+        # it's in an installed application, there's no hope.
+        try:
+            here = os.path.abspath(__file__)
+        except NameError:
+            # py2exe/bbfreeze/non-CPython don't have __file__
+            return {} # without __file__, we have no hope
+        # versionfile_source is the relative path from the top of the source
+        # tree to _version.py. Invert this to find the root from __file__.
+        root = here
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        # we're running from versioneer.py, which means we're running from
+        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
+        here = os.path.abspath(sys.argv[0])
+        root = os.path.dirname(here)
+
+    # Source tarballs conventionally unpack into a directory that includes
+    # both the project name and a version string.
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
+                  (root, dirname, parentdir_prefix))
+        return None
+    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+
+tag_prefix = "nilmdb-"
+parentdir_prefix = "nilmdb-"
+versionfile_source = "nilmdb/_version.py"
+
+def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
+    variables = { "refnames": git_refnames, "full": git_full }
+    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
+    if not ver:
+        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
+    if not ver:
+        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
+                                      verbose)
+    if not ver:
+        ver = default
+    return ver
+

nilmdb/client/__init__.py

@@ -1,4 +1,4 @@
 """nilmdb.client"""
 
-from .client import Client
+from nilmdb.client.client import Client
-from .errors import *
+from nilmdb.client.errors import ClientError, ServerError, Error

nilmdb/client/client.py

@@ -2,37 +2,47 @@
 
 """Class for performing HTTP client requests via libcurl"""
 
-import nilmdb
 import nilmdb.utils
 import nilmdb.client.httpclient
-from nilmdb.utils.printf import *
+from nilmdb.client.errors import ClientError
 
-import time
-import sys
 import re
-import os
+import time
 import simplejson as json
-import itertools
+import contextlib
 
-version = "1.0"
+from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
 
-def float_to_string(f):
+def extract_timestamp(line):
-    # Use repr to maintain full precision in the string output.
+    """Extract just the timestamp from a line of data text"""
-    return repr(float(f))
+    return string_to_timestamp(line.split()[0])
 
 class Client(object):
     """Main client interface to the Nilm database."""
 
-    client_version = version
+    def __init__(self, url, post_json = False):
+        """Initialize client with given URL.  If post_json is true,
+        POST requests are sent with Content-Type 'application/json'
+        instead of the default 'x-www-form-urlencoded'."""
+        self.http = nilmdb.client.httpclient.HTTPClient(url, post_json)
+        self.post_json = post_json
 
-    def __init__(self, url):
+    # __enter__/__exit__ allow this class to be a context manager
-        self.http = nilmdb.client.httpclient.HTTPClient(url)
+    def __enter__(self):
+        return self
 
-    def _json_param(self, data):
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+    def _json_post_param(self, data):
         """Return compact json-encoded version of parameter"""
+        if self.post_json:
+            # If we're posting as JSON, we don't need to encode it further here
+            return data
         return json.dumps(data, separators=(',',':'))
 
     def close(self):
+        """Close the connection; safe to call multiple times"""
         self.http.close()
 
     def geturl(self):
@@ -43,21 +53,25 @@ class Client(object):
         """Return server version"""
         return self.http.get("version")
 
-    def dbpath(self):
+    def dbinfo(self):
-        """Return server database path"""
+        """Return server database info (path, size, free space)
-        return self.http.get("dbpath")
+        as a dictionary."""
+        return self.http.get("dbinfo")
 
-    def dbsize(self):
+    def stream_list(self, path = None, layout = None, extended = False):
-        """Return server database size as human readable string"""
-        return self.http.get("dbsize")
-
-    def stream_list(self, path = None, layout = None):
         params = {}
         if path is not None:
             params["path"] = path
         if layout is not None:
             params["layout"] = layout
-        return self.http.get("stream/list", params)
+        if extended:
+            params["extended"] = 1
+        def sort_streams_nicely(x):
+            """Human-friendly sort (/stream/2 before /stream/10)"""
+            num = lambda t: int(t) if t.isdigit() else t
+            key = lambda k: [ num(c) for c in re.split('([0-9]+)', k[0]) ]
+            return sorted(x, key = key)
+        return sort_streams_nicely(self.http.get("stream/list", params))
 
     def stream_get_metadata(self, path, keys = None):
         params = { "path": path }
@@ -70,28 +84,34 @@ class Client(object):
         metadata."""
         params = {
             "path": path,
-            "data": self._json_param(data)
+            "data": self._json_post_param(data)
             }
-        return self.http.get("stream/set_metadata", params)
+        return self.http.post("stream/set_metadata", params)
 
     def stream_update_metadata(self, path, data):
         """Update stream metadata from a dictionary"""
         params = {
             "path": path,
-            "data": self._json_param(data)
+            "data": self._json_post_param(data)
             }
-        return self.http.get("stream/update_metadata", params)
+        return self.http.post("stream/update_metadata", params)
 
     def stream_create(self, path, layout):
         """Create a new stream"""
         params = { "path": path,
                    "layout" : layout }
-        return self.http.get("stream/create", params)
+        return self.http.post("stream/create", params)
 
     def stream_destroy(self, path):
-        """Delete stream and its contents"""
+        """Delete stream.  Fails if any data is still present."""
         params = { "path": path }
-        return self.http.get("stream/destroy", params)
+        return self.http.post("stream/destroy", params)
+
+    def stream_rename(self, oldpath, newpath):
+        """Rename a stream."""
+        params = { "oldpath": oldpath,
+                   "newpath": newpath }
+        return self.http.post("stream/rename", params)
 
     def stream_remove(self, path, start = None, end = None):
         """Remove data from the specified time range"""
@@ -99,113 +119,338 @@ class Client(object):
             "path": path
         }
         if start is not None:
-            params["start"] = float_to_string(start)
+            params["start"] = timestamp_to_string(start)
         if end is not None:
-            params["end"] = float_to_string(end)
+            params["end"] = timestamp_to_string(end)
-        return self.http.get("stream/remove", params)
+        return self.http.post("stream/remove", params)
+
+    @contextlib.contextmanager
+    def stream_insert_context(self, path, start = None, end = None):
+        """Return a context manager that allows data to be efficiently
+        inserted into a stream in a piecewise manner.  Data is
+        provided as ASCII lines, and is aggregated and sent to the
+        server in larger or smaller chunks as necessary.  Data lines
+        must match the database layout for the given path, and end
+        with a newline.
+
+        Example:
+          with client.stream_insert_context('/path', start, end) as ctx:
+            ctx.insert('1234567890.0 1 2 3 4\\n')
+            ctx.insert('1234567891.0 1 2 3 4\\n')
+
+        For more details, see help for nilmdb.client.client.StreamInserter
+
+        This may make multiple requests to the server, if the data is
+        large enough or enough time has passed between insertions.
+        """
+        ctx = StreamInserter(self, path, start, end)
+        yield ctx
+        ctx.finalize()
 
     def stream_insert(self, path, data, start = None, end = None):
-        """Insert data into a stream.  data should be a file-like object
+        """Insert rows of data into a stream.  data should be a string
-        that provides ASCII data that matches the database layout for path.
+        or iterable that provides ASCII data that matches the database
-
+        layout for path.  Data is passed through stream_insert_context,
-        start and end are the starting and ending timestamp of this
+        so it will be broken into reasonably-sized chunks and
-        stream; all timestamps t in the data must satisfy 'start <= t
+        start/end will be deduced if missing."""
-        < end'.  If left unspecified, 'start' is the timestamp of the
+        with self.stream_insert_context(path, start, end) as ctx:
-        first line of data, and 'end' is the timestamp on the last line
+            if isinstance(data, basestring):
-        of data, plus a small delta of 1μs.
+                ctx.insert(data)
-        """
-        params = { "path": path }
-
-        # See design.md for a discussion of how much data to send.
-        # These are soft limits -- actual data might be rounded up.
-        max_data = 1048576
-        max_time = 30
-        end_epsilon = 1e-6
-
-
-        def extract_timestamp(line):
-            return float(line.split()[0])
-
-        def sendit():
-            # If we have more data after this, use the timestamp of
-            # the next line as the end.  Otherwise, use the given
-            # overall end time, or add end_epsilon to the last data
-            # point.
-            if nextline:
-                block_end = extract_timestamp(nextline)
-                if end and block_end > end:
-                    # This is unexpected, but we'll defer to the server
-                    # to return an error in this case.
-                    block_end = end
-            elif end:
-                block_end = end
             else:
-                block_end = extract_timestamp(line) + end_epsilon
+                for chunk in data:
+                    ctx.insert(chunk)
+        return ctx.last_response
 
-            # Send it
+    def stream_insert_block(self, path, data, start, end, binary = False):
-            params["start"] = float_to_string(block_start)
+        """Insert a single fixed block of data into the stream.  It is
-            params["end"] = float_to_string(block_end)
+        sent directly to the server in one block with no further
-            return self.http.put("stream/insert", block_data, params)
+        processing.
 
-        clock_start = time.time()
+        If 'binary' is True, provide raw binary data in little-endian
-        block_data = ""
+        format matching the path layout, including an int64 timestamp.
-        block_start = start
+        Otherwise, provide ASCII data matching the layout."""
-        result = None
+        params = {
-        for (line, nextline) in nilmdb.utils.misc.pairwise(data):
+            "path": path,
-            # If we don't have a starting time, extract it from the first line
+            "start": timestamp_to_string(start),
-            if block_start is None:
+            "end": timestamp_to_string(end),
-                block_start = extract_timestamp(line)
+        }
+        if binary:
+            params["binary"] = 1
+        return self.http.put("stream/insert", data, params, binary = binary)
 
-            clock_elapsed = time.time() - clock_start
+    def stream_intervals(self, path, start = None, end = None, diffpath = None):
-            block_data += line
-
-            # If we have enough data, or enough time has elapsed,
-            # send this block to the server, and empty things out
-            # for the next block.
-            if (len(block_data) > max_data) or (clock_elapsed > max_time):
-                result = sendit()
-                block_start = None
-                block_data = ""
-                clock_start = time.time()
-
-        # One last block?
-        if len(block_data):
-            result = sendit()
-
-        # Return the most recent JSON result we got back, or None if
-        # we didn't make any requests.
-        return result
-
-    def stream_intervals(self, path, start = None, end = None):
         """
         Return a generator that yields each stream interval.
+
+        If 'diffpath' is not None, yields only interval ranges that are
+        present in 'path' but not in 'diffpath'.
         """
         params = {
             "path": path
         }
+        if diffpath is not None:
+            params["diffpath"] = diffpath
         if start is not None:
-            params["start"] = float_to_string(start)
+            params["start"] = timestamp_to_string(start)
         if end is not None:
-            params["end"] = float_to_string(end)
+            params["end"] = timestamp_to_string(end)
-        return self.http.get_gen("stream/intervals", params, retjson = True)
+        return self.http.get_gen("stream/intervals", params)
 
-    def stream_extract(self, path, start = None, end = None, count = False):
+    def stream_extract(self, path, start = None, end = None,
+                       count = False, markup = False, binary = False):
         """
         Extract data from a stream.  Returns a generator that yields
         lines of ASCII-formatted data that matches the database
         layout for the given path.
 
-        Specify count=True to just get a count of values rather than
+        If 'count' is True, return a count of matching data points
-        the actual data.
+        rather than the actual data.  The output format is unchanged.
+
+        If 'markup' is True, include comments in the returned data
+        that indicate interval starts and ends.
+
+        If 'binary' is True, return chunks of raw binary data, rather
+        than lines of ASCII-formatted data.  Raw binary data is
+        little-endian and matches the database types (including an
+        int64 timestamp).
         """
         params = {
             "path": path,
         }
         if start is not None:
-            params["start"] = float_to_string(start)
+            params["start"] = timestamp_to_string(start)
         if end is not None:
-            params["end"] = float_to_string(end)
+            params["end"] = timestamp_to_string(end)
         if count:
             params["count"] = 1
+        if markup:
+            params["markup"] = 1
+        if binary:
+            params["binary"] = 1
+        return self.http.get_gen("stream/extract", params, binary = binary)
 
-        return self.http.get_gen("stream/extract", params, retjson = False)
+    def stream_count(self, path, start = None, end = None):
+        """
+        Return the number of rows of data in the stream that satisfy
+        the given timestamps.
+        """
+        counts = list(self.stream_extract(path, start, end, count = True))
+        return int(counts[0])
+
+class StreamInserter(object):
+    """Object returned by stream_insert_context() that manages
+    the insertion of rows of data into a particular path.
+
+    The basic data flow is that we are filling a contiguous interval
+    on the server, with no gaps, that extends from timestamp 'start'
+    to timestamp 'end'.  Data timestamps satisfy 'start <= t < end'.
+
+    Data is provided to .insert() as ASCII formatted data separated by
+    newlines.  The chunks of data passed to .insert() do not need to
+    match up with the newlines; less or more than one line can be passed.
+
+    1. The first inserted line begins a new interval that starts at
+    'start'.  If 'start' is not given, it is deduced from the first
+    line's timestamp.
+
+    2. Subsequent lines go into the same contiguous interval.  As lines
+    are inserted, this routine may make multiple insertion requests to
+    the server, but will structure the timestamps to leave no gaps.
+
+    3. The current contiguous interval can be completed by manually
+    calling .finalize(), which the context manager will also do
+    automatically.  This will send any remaining data to the server,
+    using the 'end' timestamp to end the interval.  If no 'end'
+    was provided, it is deduced from the last timestamp seen,
+    plus a small delta.
+
+    After a .finalize(), inserting new data goes back to step 1.
+
+    .update_start() can be called before step 1 to change the start
+    time for the interval.  .update_end() can be called before step 3
+    to change the end time for the interval.
+    """
+
+    # See design.md for a discussion of how much data to send.  This
+    # is a soft limit -- we might send up to twice as much or so
+    _max_data = 2 * 1024 * 1024
+    _max_data_after_send = 64 * 1024
+
+    def __init__(self, client, path, start, end):
+        """'client' is the client object.  'path' is the database
+        path to insert to.  'start' and 'end' are used for the first
+        contiguous interval and may be None."""
+        self.last_response = None
+
+        self._client = client
+        self._path = path
+
+        # Start and end for the overall contiguous interval we're
+        # filling
+        self._interval_start = start
+        self._interval_end = end
+
+        # Current data we're building up to send.  Each string
+        # goes into the array, and gets joined all at once.
+        self._block_data = []
+        self._block_len = 0
+
+    def insert(self, data):
+        """Insert a chunk of ASCII formatted data in string form.  The
+        overall data must consist of lines terminated by '\\n'."""
+        length = len(data)
+        maxdata = self._max_data
+
+        if length > maxdata:
+            # This could make our buffer more than twice what we
+            # wanted to send, so split it up.  This is a bit
+            # inefficient, but the user really shouldn't be providing
+            # this much data at once.
+            for cut in range(0, length, maxdata):
+                self.insert(data[cut:(cut + maxdata)])
+            return
+
+        # Append this string to our list
+        self._block_data.append(data)
+        self._block_len += length
+
+        # Send the block once we have enough data
+        if self._block_len >= maxdata:
+            self._send_block(final = False)
+            if self._block_len >= self._max_data_after_send: # pragma: no cover
+                raise ValueError("too much data left over after trying"
+                                 " to send intermediate block; is it"
+                                 " missing newlines or malformed?")
+
+    def update_start(self, start):
+        """Update the start time for the next contiguous interval.
+        Call this before starting to insert data for a new interval,
+        for example, after .finalize()"""
+        self._interval_start = start
+
+    def update_end(self, end):
+        """Update the end time for the current contiguous interval.
+        Call this before .finalize()"""
+        self._interval_end = end
+
+    def finalize(self):
+        """Stop filling the current contiguous interval.
+        All outstanding data will be sent, and the interval end
+        time of the interval will be taken from the 'end' argument
+        used when initializing this class, or the most recent
+        value passed to update_end(), or the last timestamp plus
+        a small epsilon value if no other endpoint was provided.
+
+        If more data is inserted after a finalize(), it will become
+        part of a new interval and there may be a gap left in-between."""
+        self._send_block(final = True)
+
+    def send(self):
+        """Send any data that we might have buffered up.  Does not affect
+        any other treatment of timestamps or endpoints."""
+        self._send_block(final = False)
+
+    def _get_first_noncomment(self, block):
+        """Return the (start, end) indices of the first full line in
+        block that isn't a comment, or raise IndexError if
+        there isn't one."""
+        start = 0
+        while True:
+            end = block.find('\n', start)
+            if end < 0:
+                raise IndexError
+            if block[start] != '#':
+                return (start, (end + 1))
+            start = end + 1
+
+    def _get_last_noncomment(self, block):
+        """Return the (start, end) indices of the last full line in
+        block[:length] that isn't a comment, or raise IndexError if
+        there isn't one."""
+        end = block.rfind('\n')
+        if end <= 0:
+            raise IndexError
+        while True:
+            start = block.rfind('\n', 0, end)
+            if block[start + 1] != '#':
+                return ((start + 1), end)
+            if start == -1:
+                raise IndexError
+            end = start
+
+    def _send_block(self, final = False):
+        """Send data currently in the block.  The data sent will
+        consist of full lines only, so some might be left over."""
+        # Build the full string to send
+        block = "".join(self._block_data)
+
+        start_ts = self._interval_start
+        if start_ts is None:
+            # Pull start from the first line
+            try:
+                (spos, epos) = self._get_first_noncomment(block)
+                start_ts = extract_timestamp(block[spos:epos])
+            except (ValueError, IndexError):
+                pass # no timestamp is OK, if we have no data
+
+        if final:
+            # For a final block, it must end in a newline, and the
+            # ending timestamp is either the user-provided end,
+            # or the timestamp of the last line plus epsilon.
+            end_ts = self._interval_end
+            try:
+                if block[-1] != '\n':
+                    raise ValueError("final block didn't end with a newline")
+                if end_ts is None:
+                    (spos, epos) = self._get_last_noncomment(block)
+                    end_ts = extract_timestamp(block[spos:epos])
+                    end_ts += nilmdb.utils.time.epsilon
+            except (ValueError, IndexError):
+                pass # no timestamp is OK, if we have no data
+            self._block_data = []
+            self._block_len = 0
+
+            # Next block is completely fresh
+            self._interval_start = None
+            self._interval_end = None
+        else:
+            # An intermediate block, e.g. "line1\nline2\nline3\nline4"
+            # We need to save "line3\nline4" for the next block, and
+            # use the timestamp from "line3" as the ending timestamp
+            # for this one.
+            try:
+                (spos, epos) = self._get_last_noncomment(block)
+                end_ts = extract_timestamp(block[spos:epos])
+            except (ValueError, IndexError):
+                # If we found no timestamp, give up; we could send this
+                # block later when we have more data.
+                return
+            if spos == 0:
+                # Not enough data to send an intermediate block
+                return
+            if self._interval_end is not None and end_ts > self._interval_end:
+                # User gave us bad endpoints; send it anyway, and let
+                # the server complain so that the error is the same
+                # as if we hadn't done this chunking.
+                end_ts = self._interval_end
+            self._block_data = [ block[spos:] ]
+            self._block_len = (epos - spos)
+            block = block[:spos]
+
+            # Next block continues where this one ended
+            self._interval_start = end_ts
+
+        # Double check endpoints
+        if start_ts is None or end_ts is None:
+            # If the block has no non-comment lines, it's OK
+            try:
+                self._get_first_noncomment(block)
+            except IndexError:
+                return
+            raise ClientError("have data to send, but no start/end times")
+
+        # Send it
+        self.last_response = self._client.stream_insert_block(
+            self._path, block, start_ts, end_ts, binary = False)
+
+        return

nilmdb/client/httpclient.py

@@ -1,51 +1,41 @@
 """HTTP client library"""
 
-import nilmdb
 import nilmdb.utils
-from nilmdb.utils.printf import *
+from nilmdb.client.errors import ClientError, ServerError, Error
-from nilmdb.client.errors import *
 
-import time
-import sys
-import re
-import os
 import simplejson as json
 import urlparse
-import pycurl
+import requests
-import cStringIO
 
 class HTTPClient(object):
     """Class to manage and perform HTTP requests from the client"""
-    def __init__(self, baseurl = ""):
+    def __init__(self, baseurl = "", post_json = False):
         """If baseurl is supplied, all other functions that take
         a URL can be given a relative URL instead."""
         # Verify / clean up URL
         reparsed = urlparse.urlparse(baseurl).geturl()
         if '://' not in reparsed:
             reparsed = urlparse.urlparse("http://" + baseurl).geturl()
-        self.baseurl = reparsed
+        self.baseurl = reparsed.rstrip('/') + '/'
-        self.curl = pycurl.Curl()
-        self.curl.setopt(pycurl.SSL_VERIFYHOST, 2)
-        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
-        self.curl.setopt(pycurl.MAXREDIRS, 5)
-        self._setup_url()
 
-    def _setup_url(self, url = "", params = ""):
+        # Build Requests session object, enable SSL verification
-        url = urlparse.urljoin(self.baseurl, url)
+        self.session = requests.Session()
-        if params:
+        self.session.verify = True
-            url = urlparse.urljoin(
-                url, "?" + nilmdb.utils.urllib.urlencode(params))
-        self.curl.setopt(pycurl.URL, url)
-        self.url = url
 
-    def _check_error(self, body = None):
+        # Saved response, so that tests can verify a few things.
-        code = self.curl.getinfo(pycurl.RESPONSE_CODE)
+        self._last_response = {}
-        if code == 200:
+
-            return
+        # Whether to send application/json POST bodies (versus
-        # Default variables for exception
+        # x-www-form-urlencoded)
-        args = { "url" : self.url,
+        self.post_json = post_json
+
+    def _handle_error(self, url, code, body):
+        # Default variables for exception.  We use the entire body as
+        # the default message, in case we can't extract it from a JSON
+        # response.
+        args = { "url" : url,
                  "status" : str(code),
-                 "message" : None,
+                 "message" : body,
                  "traceback" : None }
         try:
             # Fill with server-provided data if we can
@@ -67,133 +57,85 @@ class HTTPClient(object):
             else:
                 raise Error(**args)
 
-    def _req_generator(self, url, params):
-        """
-        Like self._req(), but runs the perform in a separate thread.
-        It returns a generator that spits out arbitrary-sized chunks
-        of the resulting data, instead of using the WRITEFUNCTION
-        callback.
-        """
-        self._setup_url(url, params)
-        self._status = None
-        error_body = ""
-        self._headers = ""
-        def header_callback(data):
-            if self._status is None:
-                self._status = int(data.split(" ")[1])
-            self._headers += data
-        self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
-        def func(callback):
-            self.curl.setopt(pycurl.WRITEFUNCTION, callback)
-            self.curl.perform()
-        try:
-            with nilmdb.utils.Iteratorizer(func, curl_hack = True) as it:
-                for i in it:
-                    if self._status == 200:
-                        # If we had a 200 response, yield the data to caller.
-                        yield i
-                    else:
-                        # Otherwise, collect it into an error string.
-                        error_body += i
-        except pycurl.error as e:
-            raise ServerError(status = "502 Error",
-                              url = self.url,
-                              message = e[1])
-        # Raise an exception if there was an error
-        self._check_error(error_body)
-
-    def _req(self, url, params):
-        """
-        GET or POST that returns raw data.  Returns the body
-        data as a string, or raises an error if it contained an error.
-        """
-        self._setup_url(url, params)
-        body = cStringIO.StringIO()
-        self.curl.setopt(pycurl.WRITEFUNCTION, body.write)
-        self._headers = ""
-        def header_callback(data):
-            self._headers += data
-        self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
-        try:
-            self.curl.perform()
-        except pycurl.error as e:
-            raise ServerError(status = "502 Error",
-                              url = self.url,
-                              message = e[1])
-        body_str = body.getvalue()
-        # Raise an exception if there was an error
-        self._check_error(body_str)
-        return body_str
-
     def close(self):
-        self.curl.close()
+        self.session.close()
 
-    def _iterate_lines(self, it):
+    def _do_req(self, method, url, query_data, body_data, stream, headers):
+        url = urlparse.urljoin(self.baseurl, url)
+        try:
+            response = self.session.request(method, url,
+                                            params = query_data,
+                                            data = body_data,
+                                            stream = stream,
+                                            headers = headers)
+        except requests.RequestException as e:
+            raise ServerError(status = "502 Error", url = url,
+                              message = str(e.message))
+        if response.status_code != 200:
+            self._handle_error(url, response.status_code, response.content)
+        self._last_response = response
+        if response.headers["content-type"] in ("application/json",
+                                                "application/x-json-stream"):
+            return (response, True)
+        else:
+            return (response, False)
+
+    # Normal versions that return data directly
+    def _req(self, method, url, query = None, body = None, headers = None):
         """
-        Given an iterator that returns arbitrarily-sized chunks
+        Make a request and return the body data as a string or parsed
-        of data, return '\n'-delimited lines of text
+        JSON object, or raise an error if it contained an error.
         """
-        partial = ""
+        (response, isjson) = self._do_req(method, url, query, body,
-        for chunk in it:
+                                          stream = False, headers = headers)
-            partial += chunk
+        if isjson:
-            lines = partial.split("\n")
+            return json.loads(response.content)
-            for line in lines[0:-1]:
+        return response.content
-                yield line
-            partial = lines[-1]
-        if partial != "":
-            yield partial
 
-    # Non-generator versions
+    def get(self, url, params = None):
-    def _doreq(self, url, params, retjson):
+        """Simple GET (parameters in URL)"""
+        return self._req("GET", url, params, None)
+
+    def post(self, url, params = None):
+        """Simple POST (parameters in body)"""
+        if self.post_json:
+            return self._req("POST", url, None,
+                             json.dumps(params),
+                             { 'Content-type': 'application/json' })
+        else:
+            return self._req("POST", url, None, params)
+
+    def put(self, url, data, params = None, binary = False):
+        """Simple PUT (parameters in URL, data in body)"""
+        if binary:
+            h = { 'Content-type': 'application/octet-stream' }
+        else:
+            h = { 'Content-type': 'text/plain; charset=utf-8' }
+        return self._req("PUT", url, query = params, body = data, headers = h)
+
+    # Generator versions that return data one line at a time.
+    def _req_gen(self, method, url, query = None, body = None,
+                 headers = None, binary = False):
         """
-        Perform a request, and return the body.
+        Make a request and return a generator that gives back strings
-
+        or JSON decoded lines of the body data, or raise an error if
-        url: URL to request (relative to baseurl)
+        it contained an eror.
-        params: dictionary of query parameters
-        retjson: expect JSON and return python objects instead of string
         """
-        out = self._req(url, params)
+        (response, isjson) = self._do_req(method, url, query, body,
-        if retjson:
+                                          stream = True, headers = headers)
-            return json.loads(out)
+        if binary:
-        return out
+            for chunk in response.iter_content(chunk_size = 65536):
-
+                yield chunk
-    def get(self, url, params = None, retjson = True):
+        elif isjson:
-        """Simple GET"""
+            for line in response.iter_lines():
-        self.curl.setopt(pycurl.UPLOAD, 0)
-        return self._doreq(url, params, retjson)
-
-    def put(self, url, postdata, params = None, retjson = True):
-        """Simple PUT"""
-        self.curl.setopt(pycurl.UPLOAD, 1)
-        self._setup_url(url, params)
-        data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.READFUNCTION, data.read)
-        return self._doreq(url, params, retjson)
-
-    # Generator versions
-    def _doreq_gen(self, url, params, retjson):
-        """
-        Perform a request, and return lines of the body in a generator.
-
-        url: URL to request (relative to baseurl)
-        params: dictionary of query parameters
-        retjson: expect JSON and yield python objects instead of strings
-        """
-        for line in self._iterate_lines(self._req_generator(url, params)):
-            if retjson:
                 yield json.loads(line)
-            else:
+        else:
+            for line in response.iter_lines():
                 yield line
 
-    def get_gen(self, url, params = None, retjson = True):
+    def get_gen(self, url, params = None, binary = False):
-        """Simple GET, returning a generator"""
+        """Simple GET (parameters in URL) returning a generator"""
-        self.curl.setopt(pycurl.UPLOAD, 0)
+        return self._req_gen("GET", url, params, binary = binary)
-        return self._doreq_gen(url, params, retjson)
 
-    def put_gen(self, url, postdata, params = None, retjson = True):
+    # Not much use for a POST or PUT generator, since they don't
-        """Simple PUT, returning a generator"""
+    # return much data.
-        self.curl.setopt(pycurl.UPLOAD, 1)
-        self._setup_url(url, params)
-        data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.READFUNCTION, data.read)
-        return self._doreq_gen(url, params, retjson)

nilmdb/client/numpyclient.py

@@ -0,0 +1,259 @@
+# -*- coding: utf-8 -*-
+
+"""Provide a NumpyClient class that is based on normal Client, but has
+additional methods for extracting and inserting data via Numpy arrays."""
+
+import nilmdb.utils
+import nilmdb.client.client
+import nilmdb.client.httpclient
+from nilmdb.client.errors import ClientError
+
+import contextlib
+from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
+
+import numpy
+import cStringIO
+
+def layout_to_dtype(layout):
+    ltype = layout.split('_')[0]
+    lcount = int(layout.split('_')[1])
+    if ltype.startswith('int'):
+        atype = '<i' + str(int(ltype[3:]) / 8)
+    elif ltype.startswith('uint'):
+        atype = '<u' + str(int(ltype[4:]) / 8)
+    elif ltype.startswith('float'):
+        atype = '<f' + str(int(ltype[5:]) / 8)
+    else:
+        raise ValueError("bad layout")
+    return numpy.dtype([('timestamp', '<i8'), ('data', atype, lcount)])
+
+class NumpyClient(nilmdb.client.client.Client):
+    """Subclass of nilmdb.client.Client that adds additional methods for
+    extracting and inserting data via Numpy arrays."""
+
+    def _get_dtype(self, path, layout):
+        if layout is None:
+            streams = self.stream_list(path)
+            if len(streams) != 1:
+                raise ClientError("can't get layout for path: " + path)
+            layout = streams[0][1]
+        return layout_to_dtype(layout)
+
+    def stream_extract_numpy(self, path, start = None, end = None,
+                             layout = None, maxrows = 100000,
+                             structured = False):
+        """
+        Extract data from a stream.  Returns a generator that yields
+        Numpy arrays of up to 'maxrows' of data each.
+
+        If 'layout' is None, it is read using stream_info.
+
+        If 'structured' is False, all data is converted to float64
+        and returned in a flat 2D array.  Otherwise, data is returned
+        as a structured dtype in a 1D array.
+        """
+        dtype = self._get_dtype(path, layout)
+
+        def to_numpy(data):
+            a = numpy.fromstring(data, dtype)
+            if structured:
+                return a
+            return numpy.c_[a['timestamp'], a['data']]
+
+        chunks = []
+        total_len = 0
+        maxsize = dtype.itemsize * maxrows
+        for data in self.stream_extract(path, start, end, binary = True):
+            # Add this block of binary data
+            chunks.append(data)
+            total_len += len(data)
+
+            # See if we have enough to make the requested Numpy array
+            while total_len >= maxsize:
+                assembled = "".join(chunks)
+                total_len -= maxsize
+                chunks = [ assembled[maxsize:] ]
+                block = assembled[:maxsize]
+                yield to_numpy(block)
+
+        if total_len:
+            yield to_numpy("".join(chunks))
+
+    @contextlib.contextmanager
+    def stream_insert_numpy_context(self, path, start = None, end = None,
+                                    layout = None):
+        """Return a context manager that allows data to be efficiently
+        inserted into a stream in a piecewise manner.  Data is
+        provided as Numpy arrays, and is aggregated and sent to the
+        server in larger or smaller chunks as necessary.  Data format
+        must match the database layout for the given path.
+
+        For more details, see help for
+        nilmdb.client.numpyclient.StreamInserterNumpy
+
+        If 'layout' is not None, use it as the layout rather than
+        querying the database.
+        """
+        dtype = self._get_dtype(path, layout)
+        ctx = StreamInserterNumpy(self, path, start, end, dtype)
+        yield ctx
+        ctx.finalize()
+
+    def stream_insert_numpy(self, path, data, start = None, end = None,
+                            layout = None):
+        """Insert data into a stream.  data should be a Numpy array
+        which will be passed through stream_insert_numpy_context to
+        break it into chunks etc.  See the help for that function
+        for details."""
+        with self.stream_insert_numpy_context(path, start, end, layout) as ctx:
+            if isinstance(data, numpy.ndarray):
+                ctx.insert(data)
+            else:
+                for chunk in data:
+                    ctx.insert(chunk)
+        return ctx.last_response
+
+class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
+    """Object returned by stream_insert_numpy_context() that manages
+    the insertion of rows of data into a particular path.
+
+    See help for nilmdb.client.client.StreamInserter for details.
+    The only difference is that, instead of ASCII formatted data,
+    this context manager can take Numpy arrays, which are either
+    structured (1D with complex dtype) or flat (2D with simple dtype).
+    """
+
+    # Soft limit of how many bytes to send per HTTP request.
+    _max_data = 2 * 1024 * 1024
+
+    def __init__(self, client, path, start, end, dtype):
+        """
+        'client' is the client object.  'path' is the database path
+        to insert to.  'start' and 'end' are used for the first
+        contiguous interval and may be None.  'dtype' is the Numpy
+        dtype for this stream.
+        """
+        self.last_response = None
+
+        self._dtype = dtype
+        self._client = client
+        self._path = path
+
+        # Start and end for the overall contiguous interval we're
+        # filling
+        self._interval_start = start
+        self._interval_end = end
+
+        # Max rows to send at once
+        self._max_rows = self._max_data // self._dtype.itemsize
+
+        # List of the current arrays we're building up to send
+        self._block_arrays = []
+        self._block_rows = 0
+
+    def insert(self, array):
+        """Insert Numpy data, which must match the layout type."""
+        if type(array) != numpy.ndarray:
+            array = numpy.array(array)
+        if array.ndim == 1:
+            # Already a structured array; just verify the type
+            if array.dtype != self._dtype:
+                raise ValueError("wrong dtype for 1D (structured) array")
+        elif array.ndim == 2:
+            # Convert to structured array
+            sarray = numpy.zeros(array.shape[0], dtype=self._dtype)
+            sarray['timestamp'] = array[:,0]
+            # Need the squeeze in case sarray['data'] is 1 dimensional
+            sarray['data'] = numpy.squeeze(array[:,1:])
+            array = sarray
+        else:
+            raise ValueError("wrong number of dimensions in array")
+
+        length = len(array)
+        maxrows = self._max_rows
+
+        if length == 0:
+            return
+        if length > maxrows:
+            # This is more than twice what we wanted to send, so split
+            # it up.  This is a bit inefficient, but the user really
+            # shouldn't be providing this much data at once.
+            for cut in range(0, length, maxrows):
+                self.insert(array[cut:(cut + maxrows)])
+            return
+
+        # Add this array to our list
+        self._block_arrays.append(array)
+        self._block_rows += length
+
+        # Send if it's too long
+        if self._block_rows >= maxrows:
+            self._send_block(final = False)
+
+    def _send_block(self, final = False):
+        """Send the data current stored up.  One row might be left
+        over if we need its timestamp saved."""
+
+        # Build the full array to send
+        if self._block_rows == 0:
+            array = numpy.zeros(0, dtype = self._dtype)
+        else:
+            array = numpy.hstack(self._block_arrays)
+
+        # Get starting timestamp
+        start_ts = self._interval_start
+        if start_ts is None:
+            # Pull start from the first row
+            try:
+                start_ts = array['timestamp'][0]
+            except IndexError:
+                pass # no timestamp is OK, if we have no data
+
+        # Get ending timestamp
+        if final:
+            # For a final block, the timestamp is either the
+            # user-provided end, or the timestamp of the last line
+            # plus epsilon.
+            end_ts = self._interval_end
+            if end_ts is None:
+                try:
+                    end_ts = array['timestamp'][-1]
+                    end_ts += nilmdb.utils.time.epsilon
+                except IndexError:
+                    pass # no timestamp is OK, if we have no data
+            self._block_arrays = []
+            self._block_rows = 0
+
+            # Next block is completely fresh
+            self._interval_start = None
+            self._interval_end = None
+        else:
+            # An intermediate block.  We need to save the last row
+            # for the next block, and use its timestamp as the ending
+            # timestamp for this one.
+            if len(array) < 2:
+                # Not enough data to send an intermediate block
+                return
+            end_ts = array['timestamp'][-1]
+            if self._interval_end is not None and end_ts > self._interval_end:
+                # User gave us bad endpoints; send it anyway, and let
+                # the server complain so that the error is the same
+                # as if we hadn't done this chunking.
+                end_ts = self._interval_end
+            self._block_arrays = [ array[-1:] ]
+            self._block_rows = 1
+            array = array[:-1]
+
+            # Next block continues where this one ended
+            self._interval_start = end_ts
+
+        # If we have no endpoints, it's because we had no data to send.
+        if start_ts is None or end_ts is None:
+            return
+
+        # Send it
+        data = array.tostring()
+        self.last_response = self._client.stream_insert_block(
+            self._path, data, start_ts, end_ts, binary = True)
+
+        return

nilmdb/cmdline/__init__.py

@@ -1,3 +1,3 @@
 """nilmdb.cmdline"""
 
-from .cmdline import Cmdline
+from nilmdb.cmdline.cmdline import Cmdline

nilmdb/cmdline/cmdline.py

@@ -1,24 +1,29 @@
 """Command line client functionality"""
 
-import nilmdb
+import nilmdb.client
+
 from nilmdb.utils.printf import *
 from nilmdb.utils import datetime_tz
+import nilmdb.utils.time
 
-import dateutil.parser
 import sys
-import re
+import os
 import argparse
 from argparse import ArgumentDefaultsHelpFormatter as def_form
+import signal
 
-version = "1.0"
+try: # pragma: no cover
+    import argcomplete
+except ImportError: # pragma: no cover
+    argcomplete = None
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
+subcommands = [ "help", "info", "create", "list", "metadata",
-                "remove", "destroy" ]
+                "insert", "extract", "remove", "destroy",
+                "intervals", "rename" ]
 
-# Import the subcommand modules.  Equivalent way of doing this would be
+# Import the subcommand modules
-# from . import info as cmd_info
 subcmd_mods = {}
 for cmd in subcommands:
     subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist = [ cmd ])
@@ -28,76 +33,69 @@ class JimArgumentParser(argparse.ArgumentParser):
         self.print_usage(sys.stderr)
         self.exit(2, sprintf("error: %s\n", message))
 
+class Complete(object): # pragma: no cover
+    # Completion helpers, for using argcomplete (see
+    # extras/nilmtool-bash-completion.sh)
+    def escape(self, s):
+        quote_chars = [ "\\", "\"", "'", " " ]
+        for char in quote_chars:
+            s = s.replace(char, "\\" + char)
+        return s
+
+    def none(self, prefix, parsed_args, **kwargs):
+        return []
+    rate = none
+    time = none
+    url = none
+
+    def path(self, prefix, parsed_args, **kwargs):
+        client = nilmdb.client.Client(parsed_args.url)
+        return ( self.escape(s[0])
+                 for s in client.stream_list()
+                 if s[0].startswith(prefix) )
+
+    def layout(self, prefix, parsed_args, **kwargs):
+        types = [ "int8", "int16", "int32", "int64",
+                  "uint8", "uint16", "uint32", "uint64",
+                  "float32", "float64" ]
+        layouts = []
+        for i in range(1,10):
+            layouts.extend([(t + "_" + str(i)) for t in types])
+        return ( l for l in layouts if l.startswith(prefix) )
+
+    def meta_key(self, prefix, parsed_args, **kwargs):
+        return (kv.split('=')[0] for kv
+                in self.meta_keyval(prefix, parsed_args, **kwargs))
+
+    def meta_keyval(self, prefix, parsed_args, **kwargs):
+        client = nilmdb.client.Client(parsed_args.url)
+        path = parsed_args.path
+        if not path:
+            return []
+        return ( self.escape(k + '=' + v)
+                 for (k,v) in client.stream_get_metadata(path).iteritems()
+                 if k.startswith(prefix) )
+
+
 class Cmdline(object):
 
-    def __init__(self, argv):
+    def __init__(self, argv = None):
-        self.argv = argv
+        self.argv = argv or sys.argv[1:]
         self.client = None
+        self.def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
+        self.subcmd = {}
+        self.complete = Complete()
 
     def arg_time(self, toparse):
         """Parse a time string argument"""
         try:
-            return self.parse_time(toparse).totimestamp()
+            return nilmdb.utils.time.parse_time(toparse)
         except ValueError as e:
             raise argparse.ArgumentTypeError(sprintf("%s \"%s\"",
                                                      str(e), toparse))
 
-    def parse_time(self, toparse):
+    # Set up the parser
-        """
-        Parse a free-form time string and return a datetime_tz object.
-        If the string doesn't contain a timestamp, the current local
-        timezone is assumed (e.g. from the TZ env var).
-        """
-        # If string isn't "now" and doesn't contain at least 4 digits,
-        # consider it invalid.  smartparse might otherwise accept
-        # empty strings and strings with just separators.
-        if toparse != "now" and len(re.findall(r"\d", toparse)) < 4:
-            raise ValueError("not enough digits for a timestamp")
-
-        # Try to just parse the time as given
-        try:
-            return datetime_tz.datetime_tz.smartparse(toparse)
-        except ValueError:
-            pass
-
-        # Try to extract a substring in a condensed format that we expect
-        # to see in a filename or header comment
-        res = re.search(r"(^|[^\d])("            # non-numeric or SOL
-                        r"(199\d|2\d\d\d)"       # year
-                        r"[-/]?"                 # separator
-                        r"(0[1-9]|1[012])"       # month
-                        r"[-/]?"                 # separator
-                        r"([012]\d|3[01])"       # day
-                        r"[-T ]?"                # separator
-                        r"([01]\d|2[0-3])"       # hour
-                        r"[:]?"                  # separator
-                        r"([0-5]\d)"             # minute
-                        r"[:]?"                  # separator
-                        r"([0-5]\d)?"            # second
-                        r"([-+]\d\d\d\d)?"       # timezone
-                        r")", toparse)
-        if res is not None:
-            try:
-                return datetime_tz.datetime_tz.smartparse(res.group(2))
-            except ValueError:
-                pass
-
-        # Could also try to successively parse substrings, but let's
-        # just give up for now.
-        raise ValueError("unable to parse timestamp")
-
-    def time_string(self, timestamp):
-        """
-        Convert a Unix timestamp to a string for printing, using the
-        local timezone for display (e.g. from the TZ env var).
-        """
-        dt = datetime_tz.datetime_tz.fromtimestamp(timestamp)
-        return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
-
     def parser_setup(self):
-        version_string = sprintf("nilmtool %s, client library %s",
-                                 version, nilmdb.Client.client_version)
-
         self.parser = JimArgumentParser(add_help = False,
                                         formatter_class = def_form)
 
@@ -105,22 +103,22 @@ class Cmdline(object):
         group.add_argument("-h", "--help", action='help',
                            help='show this help message and exit')
         group.add_argument("-V", "--version", action="version",
-                           version=version_string)
+                           version = nilmdb.__version__)
 
         group = self.parser.add_argument_group("Server")
         group.add_argument("-u", "--url", action="store",
-                           default="http://localhost:12380/",
+                           default=self.def_url,
-                           help="NilmDB server URL (default: %(default)s)")
+                           help="NilmDB server URL (default: %(default)s)"
+                           ).completer = self.complete.url
 
-        sub = self.parser.add_subparsers(title="Commands",
+        sub = self.parser.add_subparsers(
-                                         dest="command",
+            title="Commands", dest="command",
-                                         description="Specify --help after "
+            description="Use 'help command' or 'command --help' for more "
-                                         "the command for command-specific "
+            "details on a particular command.")
-                                         "options.")
 
         # Set up subcommands (defined in separate files)
         for cmd in subcommands:
-            subcmd_mods[cmd].setup(self, sub)
+            self.subcmd[cmd] = subcmd_mods[cmd].setup(self, sub)
 
     def die(self, formatstr, *args):
         fprintf(sys.stderr, formatstr + "\n", *args)
@@ -129,25 +127,36 @@ class Cmdline(object):
         sys.exit(-1)
 
     def run(self):
+        # Set SIGPIPE to its default handler -- we don't need Python
+        # to catch it for us.
+        try:
+            signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+        except ValueError: # pragma: no cover
+            pass
+
         # Clear cached timezone, so that we can pick up timezone changes
         # while running this from the test suite.
         datetime_tz._localtz = None
 
         # Run parser
         self.parser_setup()
+        if argcomplete: # pragma: no cover
+            argcomplete.autocomplete(self.parser)
         self.args = self.parser.parse_args(self.argv)
 
         # Run arg verify handler if there is one
         if "verify" in self.args:
             self.args.verify(self)
 
-        self.client = nilmdb.Client(self.args.url)
+        self.client = nilmdb.client.Client(self.args.url)
 
-        # Make a test connection to make sure things work
+        # Make a test connection to make sure things work,
-        try:
+        # unless the particular command requests that we don't.
-            server_version = self.client.version()
+        if "no_test_connect" not in self.args:
-        except nilmdb.client.Error as e:
+            try:
-            self.die("error connecting to server: %s", str(e))
+                server_version = self.client.version()
+            except nilmdb.client.Error as e:
+                self.die("error connecting to server: %s", str(e))
 
         # Now dispatch client request to appropriate function.  Parser
         # should have ensured that we don't have any unknown commands

nilmdb/cmdline/create.py

@@ -1,9 +1,6 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
-import textwrap
 
-from argparse import ArgumentDefaultsHelpFormatter as def_form
 from argparse import RawDescriptionHelpFormatter as raw_form
 
 def setup(self, sub):
@@ -25,9 +22,12 @@ Layout types are of the format: type_count
     cmd.set_defaults(handler = cmd_create)
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
-                       help="Path (in database) of new stream, e.g. /foo/bar")
+                       help="Path (in database) of new stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
     group.add_argument("layout",
-                       help="Layout type for new stream, e.g. float32_8")
+                       help="Layout type for new stream, e.g. float32_8",
+                       ).completer = self.complete.layout
+    return cmd
 
 def cmd_create(self):
     """Create new stream"""

nilmdb/cmdline/destroy.py

@@ -1,5 +1,4 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form
@@ -8,17 +7,27 @@ def setup(self, sub):
     cmd = sub.add_parser("destroy", help="Delete a stream and all data",
                          formatter_class = def_form,
                          description="""
-                         Destroy the stream at the specified path.  All
+                         Destroy the stream at the specified path.
-                         data and metadata related to the stream is
+                         The stream must be empty.  All metadata
-                         permanently deleted.
+                         related to the stream is permanently deleted.
                          """)
     cmd.set_defaults(handler = cmd_destroy)
+    group = cmd.add_argument_group("Options")
+    group.add_argument("-R", "--remove", action="store_true",
+                       help="Remove all data before destroying stream")
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
-                       help="Path of the stream to delete, e.g. /foo/bar")
+                       help="Path of the stream to delete, e.g. /foo/bar",
+                       ).completer = self.complete.path
+    return cmd
 
 def cmd_destroy(self):
     """Destroy stream"""
+    if self.args.remove:
+        try:
+            count = self.client.stream_remove(self.args.path)
+        except nilmdb.client.ClientError as e:
+            self.die("error removing data: %s", str(e))
     try:
         self.client.stream_destroy(self.args.path)
     except nilmdb.client.ClientError as e:

nilmdb/cmdline/extract.py

@@ -1,8 +1,6 @@
 from __future__ import print_function
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
-import sys
 
 def setup(self, sub):
     cmd = sub.add_parser("extract", help="Extract data",
@@ -14,13 +12,16 @@ def setup(self, sub):
 
     group = cmd.add_argument_group("Data selection")
     group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
+                       help="Path of stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
     group.add_argument("-s", "--start", required=True,
                        metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)")
+                       help="Starting timestamp (free-form, inclusive)",
+                       ).completer = self.complete.time
     group.add_argument("-e", "--end", required=True,
                        metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)")
+                       help="Ending timestamp (free-form, noninclusive)",
+                       ).completer = self.complete.time
 
     group = cmd.add_argument_group("Output format")
     group.add_argument("-b", "--bare", action="store_true",
@@ -28,10 +29,13 @@ def setup(self, sub):
     group.add_argument("-a", "--annotate", action="store_true",
                        help="Include comments with some information "
                        "about the stream")
+    group.add_argument("-m", "--markup", action="store_true",
+                       help="Include comments with interval starts and ends")
     group.add_argument("-T", "--timestamp-raw", action="store_true",
                        help="Show raw timestamps in annotated information")
     group.add_argument("-c", "--count", action="store_true",
                        help="Just output a count of matched data points")
+    return cmd
 
 def cmd_extract_verify(self):
     if self.args.start is not None and self.args.end is not None:
@@ -45,9 +49,9 @@ def cmd_extract(self):
     layout = streams[0][1]
 
     if self.args.timestamp_raw:
-        time_string = repr
+        time_string = nilmdb.utils.time.timestamp_to_string
     else:
-        time_string = self.time_string
+        time_string = nilmdb.utils.time.timestamp_to_human
 
     if self.args.annotate:
         printf("# path: %s\n", self.args.path)
@@ -59,7 +63,8 @@ def cmd_extract(self):
     for dataline in self.client.stream_extract(self.args.path,
                                                self.args.start,
                                                self.args.end,
-                                               self.args.count):
+                                               self.args.count,
+                                               self.args.markup):
         if self.args.bare and not self.args.count:
             # Strip timestamp (first element).  Doesn't make sense
             # if we are only returning a count.

nilmdb/cmdline/help.py

@@ -0,0 +1,26 @@
+from nilmdb.utils.printf import *
+
+import argparse
+import sys
+
+def setup(self, sub):
+    cmd = sub.add_parser("help", help="Show detailed help for a command",
+                         description="""
+                         Show help for a command. 'help command' is
+                         the same as 'command --help'.
+                         """)
+    cmd.set_defaults(handler = cmd_help)
+    cmd.set_defaults(no_test_connect = True)
+    cmd.add_argument("command", nargs="?",
+                     help="Command to get help about")
+    cmd.add_argument("rest", nargs=argparse.REMAINDER,
+                     help=argparse.SUPPRESS)
+    return cmd
+
+def cmd_help(self):
+    if self.args.command in self.subcmd:
+        self.subcmd[self.args.command].print_help()
+    else:
+        self.parser.print_help()
+
+    return

nilmdb/cmdline/info.py

@@ -1,4 +1,6 @@
+import nilmdb.client
 from nilmdb.utils.printf import *
+from nilmdb.utils import human_size
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 
@@ -10,11 +12,14 @@ def setup(self, sub):
                          version.
                          """)
     cmd.set_defaults(handler = cmd_info)
+    return cmd
 
 def cmd_info(self):
     """Print info about the server"""
-    printf("Client library version: %s\n", self.client.client_version)
+    printf("Client version: %s\n", nilmdb.__version__)
     printf("Server version: %s\n", self.client.version())
     printf("Server URL: %s\n", self.client.geturl())
-    printf("Server database path: %s\n", self.client.dbpath())
+    dbinfo = self.client.dbinfo()
-    printf("Server database size: %s\n", self.client.dbsize())
+    printf("Server database path: %s\n", dbinfo["path"])
+    printf("Server database size: %s\n", human_size(dbinfo["size"]))
+    printf("Server database free space: %s\n", human_size(dbinfo["free"]))

nilmdb/cmdline/insert.py

@@ -1,7 +1,7 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
 import nilmdb.utils.timestamper as timestamper
+import nilmdb.utils.time
 
 import sys
 
@@ -10,42 +10,71 @@ def setup(self, sub):
                          description="""
                          Insert data into a stream.
                          """)
-    cmd.set_defaults(handler = cmd_insert)
+    cmd.set_defaults(verify = cmd_insert_verify,
+                     handler = cmd_insert)
     cmd.add_argument("-q", "--quiet", action='store_true',
                      help='suppress unnecessary messages')
 
     group = cmd.add_argument_group("Timestamping",
                                    description="""
-                                   If timestamps are already provided in the
+                                   To add timestamps, specify the
-                                   input date, use --none.  Otherwise,
+                                   arguments --timestamp and --rate,
-                                   provide --start, or use --filename to
+                                   and provide a starting time.
-                                   try to deduce timestamps from the file.
-
-                                   Set the TZ environment variable to change
-                                   the default timezone.
                                    """)
 
+    group.add_argument("-t", "--timestamp", action="store_true",
+                       help="Add timestamps to each line")
     group.add_argument("-r", "--rate", type=float,
-                       help="""
+                       help="Data rate, in Hz",
-                       If needed, rate in Hz (required when using --start)
+                       ).completer = self.complete.rate
-                       """)
+
+    group = cmd.add_argument_group("Start time",
+                                   description="""
+                                   Start time may be manually
+                                   specified with --start, or guessed
+                                   from the filenames using
+                                   --filename.  Set the TZ environment
+                                   variable to change the default
+                                   timezone.""")
+
     exc = group.add_mutually_exclusive_group()
     exc.add_argument("-s", "--start",
                      metavar="TIME", type=self.arg_time,
-                     help="Starting timestamp (free-form)")
+                     help="Starting timestamp (free-form)",
+                     ).completer = self.complete.time
     exc.add_argument("-f", "--filename", action="store_true",
-                     help="""
+                     help="Use filename to determine start time")
-                     Use filenames to determine start time
+
-                     (default, if filenames are provided)
+    group = cmd.add_argument_group("End time",
-                     """)
+                                   description="""
-    exc.add_argument("-n", "--none", action="store_true",
+                                   End time for the overall stream.
-                     help="Timestamp is already present, don't add one")
+                                   (required when not using --timestamp).
+                                   Set the TZ environment
+                                   variable to change the default
+                                   timezone.""")
+    group.add_argument("-e", "--end",
+                       metavar="TIME", type=self.arg_time,
+                       help="Ending timestamp (free-form)",
+                       ).completer = self.complete.time
 
     group = cmd.add_argument_group("Required parameters")
     group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
+                       help="Path of stream, e.g. /foo/bar",
-    group.add_argument("file", nargs="*", default=['-'],
+                       ).completer = self.complete.path
-                       help="File(s) to insert (default: - (stdin))")
+    group.add_argument("file", nargs = '?', default='-',
+                       help="File to insert (default: - (stdin))")
+    return cmd
+
+def cmd_insert_verify(self):
+    if self.args.timestamp:
+        if not self.args.rate:
+            self.die("error: --rate is needed, but was not specified")
+        if not self.args.filename and self.args.start is None:
+            self.die("error: need --start or --filename when adding timestamps")
+    else:
+        if self.args.start is None or self.args.end is None:
+            self.die("error: when not adding timestamps, --start and "
+                     "--end are required")
 
 def cmd_insert(self):
     # Find requested stream
@@ -53,53 +82,50 @@ def cmd_insert(self):
     if len(streams) != 1:
         self.die("error getting stream info for path %s", self.args.path)
 
-    layout = streams[0][1]
+    arg = self.args
 
-    if self.args.start and len(self.args.file) != 1:
+    try:
-        self.die("error: --start can only be used with one input file")
+        filename = arg.file
-
-    for filename in self.args.file:
         if filename == '-':
             infile = sys.stdin
         else:
             try:
-                infile = open(filename, "r")
+                infile = open(filename, "rb")
             except IOError:
                 self.die("error opening input file %s", filename)
 
-        # Build a timestamper for this file
+        if arg.start is None:
-        if self.args.none:
+            try:
-            ts = timestamper.TimestamperNull(infile)
+                arg.start = nilmdb.utils.time.parse_time(filename)
+            except ValueError:
+                self.die("error extracting start time from filename '%s'",
+                         filename)
+
+        if arg.timestamp:
+            data = timestamper.TimestamperRate(infile, arg.start, arg.rate)
         else:
-            if self.args.start:
+            data = iter(lambda: infile.read(1048576), '')
-                start = self.args.start
-            else:
-                try:
-                    start = self.parse_time(filename)
-                except ValueError:
-                    self.die("error extracting time from filename '%s'",
-                             filename)
-
-            if not self.args.rate:
-                self.die("error: --rate is needed, but was not specified")
-            rate = self.args.rate
-
-            ts = timestamper.TimestamperRate(infile, start, rate)
 
         # Print info
-        if not self.args.quiet:
+        if not arg.quiet:
-            printf("Input file: %s\n", filename)
+            printf(" Input file: %s\n", filename)
-            printf("Timestamper: %s\n", str(ts))
+            printf(" Start time: %s\n",
+                   nilmdb.utils.time.timestamp_to_human(arg.start))
+            if arg.end:
+                printf("   End time: %s\n",
+                       nilmdb.utils.time.timestamp_to_human(arg.end))
+            if arg.timestamp:
+                printf("Timestamper: %s\n", str(data))
 
         # Insert the data
-        try:
+        self.client.stream_insert(arg.path, data, arg.start, arg.end)
-            result = self.client.stream_insert(self.args.path, ts)
+
-        except nilmdb.client.Error as e:
+    except nilmdb.client.Error as e:
-            # TODO: It would be nice to be able to offer better errors
+        # TODO: It would be nice to be able to offer better errors
-            # here, particularly in the case of overlap, which just shows
+        # here, particularly in the case of overlap, which just shows
-            # ugly bracketed ranges of 16-digit numbers and a mangled URL.
+        # ugly bracketed ranges of 16-digit numbers and a mangled URL.
-            # Need to consider adding something like e.prettyprint()
+        # Need to consider adding something like e.prettyprint()
-            # that is smarter about the contents of the error.
+        # that is smarter about the contents of the error.
-            self.die("error inserting data: %s", str(e))
+        self.die("error inserting data: %s", str(e))
 
     return

nilmdb/cmdline/intervals.py

@@ -0,0 +1,66 @@
+from nilmdb.utils.printf import *
+import nilmdb.utils.time
+
+import fnmatch
+import argparse
+from argparse import ArgumentDefaultsHelpFormatter as def_form
+
+def setup(self, sub):
+    cmd = sub.add_parser("intervals", help="List intervals",
+                         formatter_class = def_form,
+                         description="""
+                         List intervals in a stream, similar to
+                         'list --detail path'.
+
+                         If '--diff diffpath' is provided, only
+                         interval ranges that are present in 'path'
+                         and not present in 'diffpath' are printed.
+                         """)
+    cmd.set_defaults(verify = cmd_intervals_verify,
+                     handler = cmd_intervals)
+
+    group = cmd.add_argument_group("Stream selection")
+    group.add_argument("path", metavar="PATH",
+                       help="List intervals for this path",
+                       ).completer = self.complete.path
+    group.add_argument("-d", "--diff", metavar="PATH",
+                       help="Subtract intervals from this path",
+                       ).completer = self.complete.path
+
+    group = cmd.add_argument_group("Interval details")
+    group.add_argument("-s", "--start",
+                       metavar="TIME", type=self.arg_time,
+                       help="Starting timestamp for intervals "
+                       "(free-form, inclusive)",
+                       ).completer = self.complete.time
+    group.add_argument("-e", "--end",
+                       metavar="TIME", type=self.arg_time,
+                       help="Ending timestamp for intervals "
+                       "(free-form, noninclusive)",
+                       ).completer = self.complete.time
+
+    group = cmd.add_argument_group("Misc options")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps when printing times")
+
+    return cmd
+
+def cmd_intervals_verify(self):
+    if self.args.start is not None and self.args.end is not None:
+        if self.args.start >= self.args.end:
+            self.parser.error("start must precede end")
+
+def cmd_intervals(self):
+    """List intervals in a stream"""
+    if self.args.timestamp_raw:
+        time_string = nilmdb.utils.time.timestamp_to_string
+    else:
+        time_string = nilmdb.utils.time.timestamp_to_human
+
+    try:
+           for (start, end) in self.client.stream_intervals(
+               self.args.path, self.args.start, self.args.end, self.args.diff):
+               printf("[ %s -> %s ]\n", time_string(start), time_string(end))
+    except nilmdb.client.ClientError as e:
+        self.die("error listing intervals: %s", str(e))
+

nilmdb/cmdline/list.py

@@ -1,6 +1,5 @@
 from nilmdb.utils.printf import *
-import nilmdb
+import nilmdb.utils.time
-import nilmdb.client
 
 import fnmatch
 import argparse
@@ -19,23 +18,39 @@ def setup(self, sub):
 
     group = cmd.add_argument_group("Stream filtering")
     group.add_argument("-p", "--path", metavar="PATH", default="*",
-                       help="Match only this path (-p can be omitted)")
+                       help="Match only this path (-p can be omitted)",
+                       ).completer = self.complete.path
     group.add_argument("path_positional", default="*",
-                       nargs="?", help=argparse.SUPPRESS)
+                       nargs="?", help=argparse.SUPPRESS,
+                       ).completer = self.complete.path
     group.add_argument("-l", "--layout", default="*",
-                       help="Match only this stream layout")
+                       help="Match only this stream layout",
+                       ).completer = self.complete.layout
+
+    group = cmd.add_argument_group("Interval info")
+    group.add_argument("-E", "--ext", action="store_true",
+                       help="Show extended stream info, like interval "
+                       "extents and row count")
 
     group = cmd.add_argument_group("Interval details")
     group.add_argument("-d", "--detail", action="store_true",
                        help="Show available data time intervals")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps in time intervals")
     group.add_argument("-s", "--start",
                        metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)")
+                       help="Starting timestamp for intervals "
+                       "(free-form, inclusive)",
+                       ).completer = self.complete.time
     group.add_argument("-e", "--end",
                        metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)")
+                       help="Ending timestamp for intervals "
+                       "(free-form, noninclusive)",
+                       ).completer = self.complete.time
+
+    group = cmd.add_argument_group("Misc options")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps when printing times")
+
+    return cmd
 
 def cmd_list_verify(self):
     # A hidden "path_positional" argument lets the user leave off the
@@ -49,31 +64,45 @@ def cmd_list_verify(self):
             self.args.path = self.args.path_positional
 
     if self.args.start is not None and self.args.end is not None:
-        if self.args.start > self.args.end:
+        if self.args.start >= self.args.end:
-            self.parser.error("start is after end")
+            self.parser.error("start must precede end")
+
+    if self.args.start is not None or self.args.end is not None:
+        if not self.args.detail:
+            self.parser.error("--start and --end only make sense with --detail")
 
 def cmd_list(self):
     """List available streams"""
-    streams = self.client.stream_list()
+    streams = self.client.stream_list(extended = True)
 
     if self.args.timestamp_raw:
-        time_string = repr
+        time_string = nilmdb.utils.time.timestamp_to_string
     else:
-        time_string = self.time_string
+        time_string = nilmdb.utils.time.timestamp_to_human
 
-    for (path, layout) in streams:
+    for stream in streams:
+        (path, layout, int_min, int_max, rows, time) = stream[:6]
         if not (fnmatch.fnmatch(path, self.args.path) and
                 fnmatch.fnmatch(layout, self.args.layout)):
             continue
 
         printf("%s %s\n", path, layout)
-        if not self.args.detail:
-            continue
 
-        printed = False
+        if self.args.ext:
-        for (start, end) in self.client.stream_intervals(path, self.args.start,
+            if int_min is None or int_max is None:
-                                                         self.args.end):
+                printf("  interval extents: (no data)\n")
-            printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
+            else:
-            printed = True
+                printf("  interval extents: %s -> %s\n",
-        if not printed:
+                       time_string(int_min), time_string(int_max))
-            printf("  (no intervals)\n")
+            printf("        total data: %d rows, %.6f seconds\n",
+                   rows or 0,
+                   nilmdb.utils.time.timestamp_to_seconds(time or 0))
+
+        if self.args.detail:
+            printed = False
+            for (start, end) in self.client.stream_intervals(
+                path, self.args.start, self.args.end):
+                printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
+                printed = True
+            if not printed:
+                printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -9,23 +9,32 @@ def setup(self, sub):
                          a stream.
                          """,
                          usage="%(prog)s path [-g [key ...] | "
-                         "-s key=value [...] | -u key=value [...]]")
+                         "-s key=value [...] | -u key=value [...]] | "
+                         "-d [key ...]")
     cmd.set_defaults(handler = cmd_metadata)
 
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
+                       help="Path of stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
 
     group = cmd.add_argument_group("Actions")
     exc = group.add_mutually_exclusive_group()
     exc.add_argument("-g", "--get", nargs="*", metavar="key",
-                     help="Get metadata for specified keys (default all)")
+                     help="Get metadata for specified keys (default all)",
+                     ).completer = self.complete.meta_key
     exc.add_argument("-s", "--set", nargs="+", metavar="key=value",
                      help="Replace all metadata with provided "
-                     "key=value pairs")
+                     "key=value pairs",
+                     ).completer = self.complete.meta_keyval
     exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
                      help="Update metadata using provided "
-                     "key=value pairs")
+                     "key=value pairs",
+                     ).completer = self.complete.meta_keyval
+    exc.add_argument("-d", "--delete", nargs="*", metavar="key",
+                     help="Delete metadata for specified keys (default all)",
+                     ).completer = self.complete.meta_key
+    return cmd
 
 def cmd_metadata(self):
     """Manipulate metadata"""
@@ -51,6 +60,16 @@ def cmd_metadata(self):
             handler(self.args.path, data)
         except nilmdb.client.ClientError as e:
             self.die("error setting/updating metadata: %s", str(e))
+    elif self.args.delete is not None:
+        # Delete (by setting values to empty strings)
+        keys = self.args.delete or None
+        try:
+            data = self.client.stream_get_metadata(self.args.path, keys)
+            for key in data:
+                data[key] = ""
+            self.client.stream_update_metadata(self.args.path, data)
+        except nilmdb.client.ClientError as e:
+            self.die("error deleting metadata: %s", str(e))
     else:
         # Get (or unspecified)
         keys = self.args.get or None
@@ -59,7 +78,7 @@ def cmd_metadata(self):
         except nilmdb.client.ClientError as e:
             self.die("error getting metadata: %s", str(e))
         for key, value in sorted(data.items()):
-            # Omit nonexistant keys
+            # Print nonexistant keys as having empty value
             if value is None:
                 value = ""
             printf("%s=%s\n", key, value)

nilmdb/cmdline/remove.py

@@ -1,7 +1,5 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
-import sys
 
 def setup(self, sub):
     cmd = sub.add_parser("remove", help="Remove data",
@@ -9,27 +7,25 @@ def setup(self, sub):
                          Remove all data from a specified time range within a
                          stream.
                          """)
-    cmd.set_defaults(verify = cmd_remove_verify,
+    cmd.set_defaults(handler = cmd_remove)
-                     handler = cmd_remove)
 
     group = cmd.add_argument_group("Data selection")
     group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
+                       help="Path of stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
     group.add_argument("-s", "--start", required=True,
                        metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)")
+                       help="Starting timestamp (free-form, inclusive)",
+                       ).completer = self.complete.time
     group.add_argument("-e", "--end", required=True,
                        metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)")
+                       help="Ending timestamp (free-form, noninclusive)",
+                       ).completer = self.complete.time
 
     group = cmd.add_argument_group("Output format")
     group.add_argument("-c", "--count", action="store_true",
                        help="Output number of data points removed")
-
+    return cmd
-def cmd_remove_verify(self):
-    if self.args.start is not None and self.args.end is not None:
-        if self.args.start > self.args.end:
-            self.parser.error("start is after end")
 
 def cmd_remove(self):
     try:

nilmdb/cmdline/rename.py

@@ -0,0 +1,31 @@
+from nilmdb.utils.printf import *
+import nilmdb.client
+
+from argparse import ArgumentDefaultsHelpFormatter as def_form
+
+def setup(self, sub):
+    cmd = sub.add_parser("rename", help="Rename a stream",
+                         formatter_class = def_form,
+                         description="""
+                         Rename a stream.
+
+                         Only the stream's path is renamed; no
+                         metadata is changed.
+                         """)
+    cmd.set_defaults(handler = cmd_rename)
+    group = cmd.add_argument_group("Required arguments")
+    group.add_argument("oldpath",
+                       help="Old path, e.g. /foo/old",
+                       ).completer = self.complete.path
+    group.add_argument("newpath",
+                       help="New path, e.g. /foo/bar/new",
+                       ).completer = self.complete.path
+
+    return cmd
+
+def cmd_rename(self):
+    """Rename a stream"""
+    try:
+        self.client.stream_rename(self.args.oldpath, self.args.newpath)
+    except nilmdb.client.ClientError as e:
+        self.die("error renaming stream: %s", str(e))

nilmdb/scripts/__init__.py

@@ -0,0 +1 @@
+# Command line scripts

nilmdb/scripts/nilmdb_server.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python
+
+import nilmdb.server
+import argparse
+import os
+import socket
+
+def main():
+    """Main entry point for the 'nilmdb-server' command line script"""
+
+    parser = argparse.ArgumentParser(
+        description = 'Run the NilmDB server',
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument("-V", "--version", action="version",
+                        version = nilmdb.__version__)
+
+    group = parser.add_argument_group("Standard options")
+    group.add_argument('-a', '--address',
+                       help = 'Only listen on the given address',
+                       default = '0.0.0.0')
+    group.add_argument('-p', '--port', help = 'Listen on the given port',
+                       type = int, default = 12380)
+    group.add_argument('-d', '--database', help = 'Database directory',
+                       default = "./db")
+    group.add_argument('-q', '--quiet', help = 'Silence output',
+                       action = 'store_true')
+    group.add_argument('-t', '--traceback',
+                       help = 'Provide tracebacks in client errors',
+                       action = 'store_true', default = False)
+
+    group = parser.add_argument_group("Debug options")
+    group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and '
+                       'invoke interactive shell afterwards',
+                       action = 'store_true')
+
+    args = parser.parse_args()
+
+    # Create database object.  Needs to be serialized before passing
+    # to the Server.
+    db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database)
+
+    # Configure the server
+    if args.quiet:
+        embedded = True
+    else:
+        embedded = False
+    server = nilmdb.server.Server(db,
+                                  host = args.address,
+                                  port = args.port,
+                                  embedded = embedded,
+                                  force_traceback = args.traceback)
+
+    # Print info
+    if not args.quiet:
+        print "Version: %s" % nilmdb.__version__
+        print "Database: %s" % (os.path.realpath(args.database))
+        if args.address == '0.0.0.0' or args.address == '::':
+            host = socket.getfqdn()
+        else:
+            host = args.address
+        print "Server URL: http://%s:%d/" % ( host, args.port)
+        print "----"
+
+    # Run it
+    if args.yappi:
+        print "Running in yappi"
+        try:
+            import yappi
+            yappi.start()
+            server.start(blocking = True)
+        finally:
+            yappi.stop()
+            yappi.print_stats(sort_type = yappi.SORTTYPE_TTOT, limit = 50)
+            from IPython import embed
+            embed(header = "Use the yappi object to explore further, "
+                  "quit to exit")
+    else:
+        server.start(blocking = True)
+
+    # Clean up
+    if not args.quiet:
+        print "Closing database"
+        db.close()
+
+if __name__ == "__main__":
+    main()

nilmdb/scripts/nilmtool.py

@@ -0,0 +1,10 @@
+#!/usr/bin/python
+
+import nilmdb.cmdline
+
+def main():
+    """Main entry point for the 'nilmtool' command line script"""
+    nilmdb.cmdline.Cmdline().run()
+
+if __name__ == "__main__":
+    main()

nilmdb/server/__init__.py

@@ -1,15 +1,21 @@
 """nilmdb.server"""
 
+from __future__ import absolute_import
+
 # Try to set up pyximport to automatically rebuild Cython modules.  If
 # this doesn't work, it's OK, as long as the modules were built externally.
 # (e.g. python setup.py build_ext --inplace)
-try:
+try: # pragma: no cover
+    import Cython
+    import distutils.version
+    if (distutils.version.LooseVersion(Cython.__version__) <
+        distutils.version.LooseVersion("0.17")): # pragma: no cover
+        raise ImportError("Cython version too old")
     import pyximport
-    pyximport.install()
+    pyximport.install(inplace = True, build_in_temp = False)
-    import layout
+except (ImportError, TypeError): # pragma: no cover
-except: # pragma: no cover
     pass
 
-from .nilmdb import NilmDB
+from nilmdb.server.nilmdb import NilmDB
-from .server import Server
+from nilmdb.server.server import Server, wsgi_application
-from .errors import *
+from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

nilmdb/server/bulkdata.py

@@ -4,27 +4,31 @@
 # nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
 from __future__ import division
-import nilmdb
 from nilmdb.utils.printf import *
+from nilmdb.utils.time import timestamp_to_string as timestamp_to_string
+import nilmdb.utils
 
 import os
-import sys
 import cPickle as pickle
-import struct
-import fnmatch
-import mmap
 import re
+import sys
+import tempfile
+
+import nilmdb.utils.lock
+from . import rocket
 
 # Up to 256 open file descriptors at any given time.
 # These variables are global so they can be used in the decorator arguments.
 table_cache_size = 16
 fd_cache_size = 16
 
-@nilmdb.utils.must_close(wrap_verify = True)
+@nilmdb.utils.must_close(wrap_verify = False)
 class BulkData(object):
     def __init__(self, basepath, **kwargs):
         self.basepath = basepath
         self.root = os.path.join(self.basepath, "data")
+        self.lock = self.root + ".lock"
+        self.lockfile = None
 
         # Tuneables
         if "file_size" in kwargs:
@@ -43,8 +47,22 @@ class BulkData(object):
         if not os.path.isdir(self.root):
             os.mkdir(self.root)
 
+        # Create the lock
+        self.lockfile = open(self.lock, "w")
+        if not nilmdb.utils.lock.exclusive_lock(self.lockfile):
+            raise IOError('database at "' + self.basepath +
+                          '" is already locked by another process')
+
     def close(self):
         self.getnode.cache_remove_all()
+        if self.lockfile:
+            nilmdb.utils.lock.exclusive_unlock(self.lockfile)
+            self.lockfile.close()
+            try:
+                os.unlink(self.lock)
+            except OSError: # pragma: no cover
+                pass
+            self.lockfile = None
 
     def _encode_filename(self, path):
         # Encode all paths to UTF-8, regardless of sys.getfilesystemencoding(),
@@ -55,6 +73,64 @@ class BulkData(object):
             return path.encode('utf-8')
         return path
 
+    def _create_check_ospath(self, ospath):
+        if ospath[-1] == '/':
+            raise ValueError("invalid path; should not end with a /")
+        if Table.exists(ospath):
+            raise ValueError("stream already exists at this path")
+        if os.path.isdir(ospath):
+            # Look for any files in subdirectories.  Fully empty subdirectories
+            # are OK; they might be there during a rename
+            for (root, dirs, files) in os.walk(ospath):
+                if len(files):
+                    raise ValueError(
+                        "non-empty subdirs of this path already exist")
+
+    def _create_parents(self, unicodepath):
+        """Verify the path name, and create parent directories if they
+        don't exist.  Returns a list of elements that got created."""
+        path = self._encode_filename(unicodepath)
+
+        if path[0] != '/':
+            raise ValueError("paths must start with /")
+        [ group, node ] = path.rsplit("/", 1)
+        if group == '':
+            raise ValueError("invalid path; path must contain at least one "
+                             "folder")
+        if node == '':
+            raise ValueError("invalid path; should not end with a /")
+        if not Table.valid_path(path):
+            raise ValueError("path name is invalid or contains reserved words")
+
+        # Create the table's base dir.  Note that we make a
+        # distinction here between NilmDB paths (always Unix style,
+        # split apart manually) and OS paths (built up with
+        # os.path.join)
+
+        # Make directories leading up to this one
+        elements = path.lstrip('/').split('/')
+        made_dirs = []
+        try:
+            # Make parent elements
+            for i in range(len(elements)):
+                ospath = os.path.join(self.root, *elements[0:i])
+                if Table.exists(ospath):
+                    raise ValueError("path is subdir of existing node")
+                if not os.path.isdir(ospath):
+                    os.mkdir(ospath)
+                    made_dirs.append(ospath)
+        except Exception as e:
+            # Try to remove paths that we created; ignore errors
+            exc_info = sys.exc_info()
+            for ospath in reversed(made_dirs): # pragma: no cover (hard to hit)
+                try:
+                    os.rmdir(ospath)
+                except OSError:
+                    pass
+            raise exc_info[1], None, exc_info[2]
+
+        return elements
+
     def create(self, unicodepath, layout_name):
         """
         unicodepath: path to the data (e.g. u'/newton/prep').
@@ -66,64 +142,82 @@ class BulkData(object):
 
         layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
         """
-        path = self._encode_filename(unicodepath)
+        elements = self._create_parents(unicodepath)
-
-        if path[0] != '/':
-            raise ValueError("paths must start with /")
-        [ group, node ] = path.rsplit("/", 1)
-        if group == '':
-            raise ValueError("invalid path; path must contain at least one "
-                             "folder")
-
-        # Get layout, and build format string for struct module
-        try:
-            layout = nilmdb.server.layout.get_named(layout_name)
-            struct_fmt = '<d'  # Little endian, double timestamp
-            struct_mapping = {
-                "int8": 'b',
-                "uint8": 'B',
-                "int16": 'h',
-                "uint16": 'H',
-                "int32": 'i',
-                "uint32": 'I',
-                "int64": 'q',
-                "uint64": 'Q',
-                "float32": 'f',
-                "float64": 'd',
-                }
-            for n in range(layout.count):
-                struct_fmt += struct_mapping[layout.datatype]
-        except KeyError:
-            raise ValueError("no such layout, or bad data types")
-
-        # Create the table.  Note that we make a distinction here
-        # between NilmDB paths (always Unix style, split apart
-        # manually) and OS paths (built up with os.path.join)
-
-        # Make directories leading up to this one
-        elements = path.lstrip('/').split('/')
-        for i in range(len(elements)):
-            ospath = os.path.join(self.root, *elements[0:i])
-            if Table.exists(ospath):
-                raise ValueError("path is subdir of existing node")
-            if not os.path.isdir(ospath):
-                os.mkdir(ospath)
 
         # Make the final dir
         ospath = os.path.join(self.root, *elements)
-        if os.path.isdir(ospath):
+        self._create_check_ospath(ospath)
-            raise ValueError("subdirs of this path already exist")
         os.mkdir(ospath)
 
-        # Write format string to file
+        try:
-        Table.create(ospath, struct_fmt, self.file_size, self.files_per_dir)
+            # Write format string to file
+            Table.create(ospath, layout_name, self.file_size,
+                         self.files_per_dir)
 
-        # Open and cache it
+            # Open and cache it
-        self.getnode(unicodepath)
+            self.getnode(unicodepath)
+        except Exception:
+            exc_info = sys.exc_info()
+            try:
+                os.rmdir(ospath)
+            except OSError:
+                pass
+            raise exc_info[1], None, exc_info[2]
 
         # Success
         return
 
+    def _remove_leaves(self, unicodepath):
+        """Remove empty directories starting at the leaves of unicodepath"""
+        path = self._encode_filename(unicodepath)
+        elements = path.lstrip('/').split('/')
+        for i in reversed(range(len(elements))):
+            ospath = os.path.join(self.root, *elements[0:i+1])
+            try:
+                os.rmdir(ospath)
+            except OSError:
+                pass
+
+    def rename(self, oldunicodepath, newunicodepath):
+        """Move entire tree from 'oldunicodepath' to
+        'newunicodepath'"""
+        oldpath = self._encode_filename(oldunicodepath)
+        newpath = self._encode_filename(newunicodepath)
+
+        # Get OS paths
+        oldelements = oldpath.lstrip('/').split('/')
+        oldospath = os.path.join(self.root, *oldelements)
+        newelements = newpath.lstrip('/').split('/')
+        newospath = os.path.join(self.root, *newelements)
+
+        # Basic checks
+        if oldospath == newospath:
+            raise ValueError("old and new paths are the same")
+
+        # Move the table to a temporary location
+        tmpdir = tempfile.mkdtemp(prefix = "rename-", dir = self.root)
+        tmppath = os.path.join(tmpdir, "table")
+        os.rename(oldospath, tmppath)
+
+        try:
+            # Check destination path
+            self._create_check_ospath(newospath)
+
+            # Create parent dirs for new location
+            self._create_parents(newunicodepath)
+
+            # Move table into new location
+            os.rename(tmppath, newospath)
+        except Exception:
+            # On failure, move the table back to original path
+            os.rename(tmppath, oldospath)
+            os.rmdir(tmpdir)
+            raise
+
+        # Prune old dirs
+        self._remove_leaves(oldunicodepath)
+        os.rmdir(tmpdir)
+
     def destroy(self, unicodepath):
         """Fully remove all data at a particular path.  No way to undo
         it!  The group/path structure is removed, too."""
@@ -145,13 +239,8 @@ class BulkData(object):
             for name in dirs:
                 os.rmdir(os.path.join(root, name))
 
-        # Remove empty parent directories
+        # Remove leftover empty directories
-        for i in reversed(range(len(elements))):
+        self._remove_leaves(unicodepath)
-            ospath = os.path.join(self.root, *elements[0:i+1])
-            try:
-                os.rmdir(ospath)
-            except OSError:
-                break
 
     # Cache open tables
     @nilmdb.utils.lru_cache(size = table_cache_size,
@@ -164,57 +253,70 @@ class BulkData(object):
         ospath = os.path.join(self.root, *elements)
         return Table(ospath)
 
-@nilmdb.utils.must_close(wrap_verify = True)
+@nilmdb.utils.must_close(wrap_verify = False)
 class Table(object):
     """Tools to help access a single table (data at a specific OS path)."""
     # See design.md for design details
 
     # Class methods, to help keep format details in this class.
+    @classmethod
+    def valid_path(cls, root):
+        """Return True if a root path is a valid name"""
+        return "_format" not in root.split("/")
+
     @classmethod
     def exists(cls, root):
         """Return True if a table appears to exist at this OS path"""
         return os.path.isfile(os.path.join(root, "_format"))
 
     @classmethod
-    def create(cls, root, struct_fmt, file_size, files_per_dir):
+    def create(cls, root, layout, file_size, files_per_dir):
-        """Initialize a table at the given OS path.
+        """Initialize a table at the given OS path with the
-        'struct_fmt' is a Struct module format description"""
+        given layout string"""
 
         # Calculate rows per file so that each file is approximately
         # file_size bytes.
-        packer = struct.Struct(struct_fmt)
+        rkt = rocket.Rocket(layout, None)
-        rows_per_file = max(file_size // packer.size, 1)
+        rows_per_file = max(file_size // rkt.binary_size, 1)
+        rkt.close()
 
-        format = { "rows_per_file": rows_per_file,
+        fmt = { "rows_per_file": rows_per_file,
-                   "files_per_dir": files_per_dir,
+                "files_per_dir": files_per_dir,
-                   "struct_fmt": struct_fmt,
+                "layout": layout,
-                   "version": 1 }
+                "version": 3 }
         with open(os.path.join(root, "_format"), "wb") as f:
-            pickle.dump(format, f, 2)
+            pickle.dump(fmt, f, 2)
 
     # Normal methods
     def __init__(self, root):
         """'root' is the full OS path to the directory of this table"""
         self.root = root
 
-        # Load the format and build packer
+        # Load the format
         with open(os.path.join(self.root, "_format"), "rb") as f:
-            format = pickle.load(f)
+            fmt = pickle.load(f)
 
-        if format["version"] != 1: # pragma: no cover (just future proofing)
+        if fmt["version"] != 3: # pragma: no cover
-            raise NotImplementedError("version " + format["version"] +
+            # Old versions used floating point timestamps, which aren't
-                                      " bulk data store not supported")
+            # valid anymore.
+            raise NotImplementedError("old version " + str(fmt["version"]) +
+                                      " bulk data store is not supported")
 
-        self.rows_per_file = format["rows_per_file"]
+        self.rows_per_file = fmt["rows_per_file"]
-        self.files_per_dir = format["files_per_dir"]
+        self.files_per_dir = fmt["files_per_dir"]
-        self.packer = struct.Struct(format["struct_fmt"])
+        self.layout = fmt["layout"]
-        self.file_size = self.packer.size * self.rows_per_file
+
+        # Use rocket to get row size and file size
+        rkt = rocket.Rocket(self.layout, None)
+        self.row_size = rkt.binary_size
+        self.file_size = rkt.binary_size * self.rows_per_file
+        rkt.close()
 
         # Find nrows
         self.nrows = self._get_nrows()
 
     def close(self):
-        self.mmap_open.cache_remove_all()
+        self.file_open.cache_remove_all()
 
     # Internal helpers
     def _get_nrows(self):
@@ -263,124 +365,186 @@ class Table(object):
         # will just get longer but will still sort correctly.
         dirname = sprintf("%04x", filenum // self.files_per_dir)
         filename = sprintf("%04x", filenum % self.files_per_dir)
-        offset = (row % self.rows_per_file) * self.packer.size
+        offset = (row % self.rows_per_file) * self.row_size
         count = self.rows_per_file - (row % self.rows_per_file)
         return (dirname, filename, offset, count)
 
     def _row_from_offset(self, subdir, filename, offset):
         """Return the row number that corresponds to the given
         'subdir/filename' and byte-offset within that file."""
-        if (offset % self.packer.size) != 0: # pragma: no cover; shouldn't occur
+        if (offset % self.row_size) != 0: # pragma: no cover
+            # this shouldn't occur, unless there is some corruption somewhere
             raise ValueError("file offset is not a multiple of data size")
         filenum = int(subdir, 16) * self.files_per_dir + int(filename, 16)
-        row = (filenum * self.rows_per_file) + (offset // self.packer.size)
+        row = (filenum * self.rows_per_file) + (offset // self.row_size)
         return row
 
+    def _remove_or_truncate_file(self, subdir, filename, offset = 0):
+        """Remove the given file, and remove the subdirectory too
+        if it's empty.  If offset is nonzero, truncate the file
+        to that size instead."""
+        # Close potentially open file in file_open LRU cache
+        self.file_open.cache_remove(self, subdir, filename)
+        if offset:
+            # Truncate it
+            with open(os.path.join(self.root, subdir, filename), "r+b") as f:
+                f.truncate(offset)
+        else:
+            # Remove file
+            os.remove(os.path.join(self.root, subdir, filename))
+            # Try deleting subdir, too
+            try:
+                os.rmdir(os.path.join(self.root, subdir))
+            except Exception:
+                pass
+
     # Cache open files
     @nilmdb.utils.lru_cache(size = fd_cache_size,
-                            keys = slice(0,3), # exclude newsize
+                            onremove = lambda f: f.close())
-                            onremove = lambda x: x.close())
+    def file_open(self, subdir, filename):
-    def mmap_open(self, subdir, filename, newsize = None):
         """Open and map a given 'subdir/filename' (relative to self.root).
-        Will be automatically closed when evicted from the cache.
+        Will be automatically closed when evicted from the cache."""
-
+        # Create path if it doesn't exist
-        If 'newsize' is provided, the file is truncated to the given
-        size before the mapping is returned.  (Note that the LRU cache
-        on this function means the truncate will only happen if the
-        object isn't already cached; mmap.resize should be used too.)"""
         try:
             os.mkdir(os.path.join(self.root, subdir))
         except OSError:
             pass
-        f = open(os.path.join(self.root, subdir, filename), "a+", 0)
+        # Return a rocket.Rocket object, which contains the open file
-        if newsize is not None:
+        return rocket.Rocket(self.layout,
-            # mmap can't map a zero-length file, so this allows the
+                             os.path.join(self.root, subdir, filename))
-            # caller to set the filesize between file creation and
-            # mmap.
-            f.truncate(newsize)
-        mm = mmap.mmap(f.fileno(), 0)
-        return mm
 
-    def mmap_open_resize(self, subdir, filename, newsize):
+    def append_data(self, data, start, end, binary = False):
-        """Open and map a given 'subdir/filename' (relative to self.root).
+        """Parse the formatted string in 'data', according to the
-        The file is resized to the given size."""
+        current layout, and append it to the table.  If any timestamps
-        # Pass new size to mmap_open
+        are non-monotonic, or don't fall between 'start' and 'end',
-        mm = self.mmap_open(subdir, filename, newsize)
+        a ValueError is raised.
-        # In case we got a cached copy, need to call mm.resize too.
-        mm.resize(newsize)
-        return mm
 
-    def append(self, data):
+        If 'binary' is True, the data should be in raw binary format
-        """Append the data and flush it to disk.
+        instead: little-endian, matching the current table's layout,
-        data is a nested Python list [[row],[row],[...]]"""
+        including the int64 timestamp.
-        remaining = len(data)
+
-        dataiter = iter(data)
+        If this function succeeds, it returns normally.  Otherwise,
-        while remaining:
+        the table is reverted back to its original state by truncating
-            # See how many rows we can fit into the current file, and open it
+        or deleting files as necessary."""
-            (subdir, fname, offset, count) = self._offset_from_row(self.nrows)
+        data_offset = 0
+        last_timestamp = nilmdb.utils.time.min_timestamp
+        tot_rows = self.nrows
+        count = 0
+        linenum = 0
+        try:
+            while data_offset < len(data):
+                # See how many rows we can fit into the current file,
+                # and open it
+                (subdir, fname, offset, count) = self._offset_from_row(tot_rows)
+                f = self.file_open(subdir, fname)
+
+                # Ask the rocket object to parse and append up to "count"
+                # rows of data, verifying things along the way.
+                try:
+                    if binary:
+                        appender = f.append_binary
+                    else:
+                        appender = f.append_string
+                    (added_rows, data_offset, last_timestamp, linenum
+                     ) = appender(count, data, data_offset, linenum,
+                                  start, end, last_timestamp)
+                except rocket.ParseError as e:
+                    (linenum, colnum, errtype, obj) = e.args
+                    if binary:
+                        where = "byte %d: " % (linenum)
+                    else:
+                        where = "line %d, column %d: " % (linenum, colnum)
+                    # Extract out the error line, add column marker
+                    try:
+                        if binary:
+                            raise IndexError
+                        bad = data.splitlines()[linenum-1]
+                        bad += '\n' + ' ' * (colnum - 1) + '^'
+                    except IndexError:
+                        bad = ""
+                    if errtype == rocket.ERR_NON_MONOTONIC:
+                        err = "timestamp is not monotonically increasing"
+                    elif errtype == rocket.ERR_OUT_OF_INTERVAL:
+                        if obj < start:
+                            err = sprintf("Data timestamp %s < start time %s",
+                                          timestamp_to_string(obj),
+                                          timestamp_to_string(start))
+                        else:
+                            err = sprintf("Data timestamp %s >= end time %s",
+                                          timestamp_to_string(obj),
+                                          timestamp_to_string(end))
+                    else:
+                        err = str(obj)
+                    raise ValueError("error parsing input data: " +
+                                     where + err + "\n" + bad)
+                tot_rows += added_rows
+        except Exception:
+            # Some failure, so try to roll things back by truncating or
+            # deleting files that we may have appended data to.
+            cleanpos = self.nrows
+            while cleanpos <= tot_rows:
+                (subdir, fname, offset, count) = self._offset_from_row(cleanpos)
+                self._remove_or_truncate_file(subdir, fname, offset)
+                cleanpos += count
+            # Re-raise original exception
+            raise
+        else:
+            # Success, so update self.nrows accordingly
+            self.nrows = tot_rows
+
+    def get_data(self, start, stop, binary = False):
+        """Extract data corresponding to Python range [n:m],
+        and returns a formatted string"""
+        if (start is None or
+            stop is None or
+            start > stop or
+            start < 0 or
+            stop > self.nrows):
+            raise IndexError("Index out of range")
+
+        ret = []
+        row = start
+        remaining = stop - start
+        while remaining > 0:
+            (subdir, filename, offset, count) = self._offset_from_row(row)
             if count > remaining:
                 count = remaining
-            newsize = offset + count * self.packer.size
+            f = self.file_open(subdir, filename)
-            mm = self.mmap_open_resize(subdir, fname, newsize)
+            if binary:
-            mm.seek(offset)
+                ret.append(f.extract_binary(offset, count))
-
+            else:
-            # Write the data
+                ret.append(f.extract_string(offset, count))
-            for i in xrange(count):
-                row = dataiter.next()
-                mm.write(self.packer.pack(*row))
             remaining -= count
-            self.nrows += count
+            row += count
+        return b"".join(ret)
 
-    def __getitem__(self, key):
+    def __getitem__(self, row):
-        """Extract data and return it.  Supports simple indexing
+        """Extract timestamps from a row, with table[n] notation."""
-        (table[n]) and range slices (table[n:m]).  Returns a nested
+        if row < 0 or row >= self.nrows:
-        Python list [[row],[row],[...]]"""
-
-        # Handle simple slices
-        if isinstance(key, slice):
-            # Fall back to brute force if the slice isn't simple
-            if ((key.step is not None and key.step != 1) or
-                key.start is None or
-                key.stop is None or
-                key.start >= key.stop or
-                key.start < 0 or
-                key.stop > self.nrows):
-                return [ self[x] for x in xrange(*key.indices(self.nrows)) ]
-
-            ret = []
-            row = key.start
-            remaining = key.stop - key.start
-            while remaining:
-                (subdir, filename, offset, count) = self._offset_from_row(row)
-                if count > remaining:
-                    count = remaining
-                mm = self.mmap_open(subdir, filename)
-                for i in xrange(count):
-                    ret.append(list(self.packer.unpack_from(mm, offset)))
-                    offset += self.packer.size
-                remaining -= count
-                row += count
-            return ret
-
-        # Handle single points
-        if key < 0 or key >= self.nrows:
             raise IndexError("Index out of range")
-        (subdir, filename, offset, count) = self._offset_from_row(key)
+        (subdir, filename, offset, count) = self._offset_from_row(row)
-        mm = self.mmap_open(subdir, filename)
+        f = self.file_open(subdir, filename)
-        # unpack_from ignores the mmap object's current seek position
+        return f.extract_timestamp(offset)
-        return list(self.packer.unpack_from(mm, offset))
 
     def _remove_rows(self, subdir, filename, start, stop):
         """Helper to mark specific rows as being removed from a
-        file, and potentially removing or truncating the file itself."""
+        file, and potentially remove or truncate the file itself."""
-        # Import an existing list of deleted rows for this file
+        # Close potentially open file in file_open LRU cache
+        self.file_open.cache_remove(self, subdir, filename)
+
+        # We keep a file like 0000.removed that contains a list of
+        # which rows have been "removed".  Note that we never have to
+        # remove entries from this list, because we never decrease
+        # self.nrows, and so we will never overwrite those locations in the
+        # file.  Only when the list covers the entire extent of the
+        # file will that file be removed.
         datafile = os.path.join(self.root, subdir, filename)
         cachefile = datafile + ".removed"
         try:
             with open(cachefile, "rb") as f:
                 ranges = pickle.load(f)
             cachefile_present = True
-        except:
+        except Exception:
             ranges = []
             cachefile_present = False
 
@@ -413,20 +577,19 @@ class Table(object):
         # are generally easier if we don't have to special-case that.
         if (len(merged) == 1 and
             merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
-            # Close potentially open file in mmap_open LRU cache
-            self.mmap_open.cache_remove(self, subdir, filename)
-
             # Delete files
-            os.remove(datafile)
             if cachefile_present:
                 os.remove(cachefile)
-
+            self._remove_or_truncate_file(subdir, filename, 0)
-            # Try deleting subdir, too
-            try:
-                os.rmdir(os.path.join(self.root, subdir))
-            except:
-                pass
         else:
+            # File needs to stick around.  This means we can get
+            # degenerate cases where we have large files containing as
+            # little as one row.  Try to punch a hole in the file,
+            # so that this region doesn't take up filesystem space.
+            offset = start * self.row_size
+            count = (stop - start) * self.row_size
+            nilmdb.utils.fallocate.punch_hole(datafile, offset, count)
+
             # Update cache.  Try to do it atomically.
             nilmdb.utils.atomic.replace_file(cachefile,
                                              pickle.dumps(merged, 2))
@@ -447,16 +610,8 @@ class Table(object):
             (subdir, filename, offset, count) = self._offset_from_row(row)
             if count > remaining:
                 count = remaining
-            row_offset = offset // self.packer.size
+            row_offset = offset // self.row_size
             # Mark the rows as being removed
             self._remove_rows(subdir, filename, row_offset, row_offset + count)
             remaining -= count
             row += count
-
-class TimestampOnlyTable(object):
-    """Helper that lets us pass a Tables object into bisect, by
-    returning only the timestamp when a particular row is requested."""
-    def __init__(self, table):
-        self.table = table
-    def __getitem__(self, index):
-        return self.table[index][0]

nilmdb/server/interval.pyx

@@ -1,5 +1,9 @@
 """Interval, IntervalSet
 
+The Interval implemented here is just like
+nilmdb.utils.interval.Interval, except implemented in Cython for
+speed.
+
 Represents an interval of time, and a set of such intervals.
 
 Intervals are half-open, ie. they include data points with timestamps
@@ -19,49 +23,44 @@ Intervals are half-open, ie. they include data points with timestamps
 # Fourth version is an optimized rb-tree that stores interval starts
 # and ends directly in the tree, like bxinterval did.
 
-cimport rbtree
+from ..utils.time import min_timestamp as nilmdb_min_timestamp
-cdef extern from "stdint.h":
+from ..utils.time import max_timestamp as nilmdb_max_timestamp
-    ctypedef unsigned long long uint64_t
+from ..utils.time import timestamp_to_string
+from ..utils.iterator import imerge
+from ..utils.interval import IntervalError
+import itertools
 
-class IntervalError(Exception):
+cimport rbtree
-    """Error due to interval overlap, etc"""
+from libc.stdint cimport uint64_t, int64_t
-    pass
+
+ctypedef int64_t timestamp_t
 
 cdef class Interval:
     """Represents an interval of time."""
 
-    cdef public double start, end
+    cdef public timestamp_t start, end
 
-    def __init__(self, double start, double end):
+    def __init__(self, timestamp_t start, timestamp_t end):
         """
-        'start' and 'end' are arbitrary floats that represent time
+        'start' and 'end' are arbitrary numbers that represent time
         """
-        if start > end:
+        if start >= end:
             # Explicitly disallow zero-width intervals (since they're half-open)
             raise IntervalError("start %s must precede end %s" % (start, end))
-        self.start = float(start)
+        self.start = start
-        self.end = float(end)
+        self.end = end
 
     def __repr__(self):
         s = repr(self.start) + ", " + repr(self.end)
         return self.__class__.__name__ + "(" + s + ")"
 
     def __str__(self):
-        return "[" + repr(self.start) + " -> " + repr(self.end) + ")"
+        return ("[" + timestamp_to_string(self.start) +
+                " -> " + timestamp_to_string(self.end) + ")")
 
     def __cmp__(self, Interval other):
         """Compare two intervals.  If non-equal, order by start then end"""
-        if not isinstance(other, Interval):
+        return cmp(self.start, other.start) or cmp(self.end, other.end)
-            raise TypeError("bad type")
-        if self.start == other.start:
-            if self.end < other.end:
-                return -1
-            if self.end > other.end:
-                return 1
-            return 0
-        if self.start < other.start:
-            return -1
-        return 1
 
     cpdef intersects(self, Interval other):
         """Return True if two Interval objects intersect"""
@@ -69,7 +68,7 @@ cdef class Interval:
             return False
         return True
 
-    cpdef subset(self, double start, double end):
+    cpdef subset(self, timestamp_t start, timestamp_t end):
         """Return a new Interval that is a subset of this one"""
         # A subclass that tracks additional data might override this.
         if start < self.start or end > self.end:
@@ -91,14 +90,14 @@ cdef class DBInterval(Interval):
         db_end = 200, db_endpos = 20000
     """
 
-    cpdef public double db_start, db_end
+    cpdef public timestamp_t db_start, db_end
     cpdef public uint64_t db_startpos, db_endpos
 
     def __init__(self, start, end,
                  db_start, db_end,
                  db_startpos, db_endpos):
         """
-        'db_start' and 'db_end' are arbitrary floats that represent
+        'db_start' and 'db_end' are arbitrary numbers that represent
         time.  They must be a strict superset of the time interval
         covered by 'start' and 'end'.  The 'db_startpos' and
         'db_endpos' are arbitrary database position indicators that
@@ -118,7 +117,7 @@ cdef class DBInterval(Interval):
         s += ", " + repr(self.db_startpos) + ", " + repr(self.db_endpos)
         return self.__class__.__name__ + "(" + s + ")"
 
-    cpdef subset(self, double start, double end):
+    cpdef subset(self, timestamp_t start, timestamp_t end):
         """
         Return a new DBInterval that is a subset of this one
         """
@@ -262,21 +261,15 @@ cdef class IntervalSet:
 
     def __and__(self, other not None):
         """
-        Compute a new IntervalSet from the intersection of two others
+        Compute a new IntervalSet from the intersection of this
+        IntervalSet with one other interval.
 
         Output intervals are built as subsets of the intervals in the
         first argument (self).
         """
         out = IntervalSet()
-
+        for i in self.intersection(other):
-        if not isinstance(other, IntervalSet):
+            out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-            for i in self.intersection(other):
-                out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-        else:
-            for x in other:
-                for i in self.intersection(x):
-                    out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-
         return out
 
     def intersection(self, Interval interval not None, orig = False):
@@ -293,23 +286,18 @@ cdef class IntervalSet:
         (potentially) subsetted to make the one that is being
         returned.
         """
-        if not isinstance(interval, Interval):
+        if orig:
-            raise TypeError("bad type")
+            for n in self.tree.intersect(interval.start, interval.end):
-        for n in self.tree.intersect(interval.start, interval.end):
+                i = n.obj
-            i = n.obj
+                subset = i.subset(max(i.start, interval.start),
-            if i:
+                                  min(i.end, interval.end))
-                if i.start >= interval.start and i.end <= interval.end:
+                yield (subset, i)
-                    if orig:
+        else:
-                        yield (i, i)
+            for n in self.tree.intersect(interval.start, interval.end):
-                    else:
+                i = n.obj
-                        yield i
+                subset = i.subset(max(i.start, interval.start),
-                else:
+                                  min(i.end, interval.end))
-                    subset = i.subset(max(i.start, interval.start),
+                yield subset
-                                      min(i.end, interval.end))
-                    if orig:
-                        yield (subset, i)
-                    else:
-                        yield subset
 
     cpdef intersects(self, Interval other):
         """Return True if this IntervalSet intersects another interval"""
@@ -318,7 +306,7 @@ cdef class IntervalSet:
                 return True
         return False
 
-    def find_end(self, double t):
+    def find_end(self, timestamp_t t):
         """
         Return an Interval from this tree that ends at time t, or
         None if it doesn't exist.

nilmdb/server/layout.pyx

@@ -1,209 +0,0 @@
-# cython: profile=False
-
-import time
-import sys
-import inspect
-import cStringIO
-import numpy as np
-
-cdef enum:
-    max_value_count = 64
-
-cimport cython
-cimport libc.stdlib
-cimport libc.stdio
-cimport libc.string
-
-class ParserError(Exception):
-    def __init__(self, line, message):
-        self.message = "line " + str(line) + ": " + message
-        Exception.__init__(self, self.message)
-
-class FormatterError(Exception):
-    pass
-
-class Layout:
-    """Represents a NILM database layout"""
-
-    def __init__(self, typestring):
-        """Initialize this Layout object to handle the specified
-        type string"""
-        try:
-            [ datatype, count ] = typestring.split("_")
-        except:
-            raise KeyError("invalid layout string")
-
-        try:
-            self.count = int(count)
-        except ValueError:
-            raise KeyError("invalid count")
-        if self.count < 1 or self.count > max_value_count:
-            raise KeyError("invalid count")
-
-        if datatype == 'uint16':
-            self.parse = self.parse_uint16
-            self.format = self.format_uint16
-        elif datatype == 'float32' or datatype == 'float64':
-            self.parse = self.parse_float64
-            self.format = self.format_float64
-        else:
-            raise KeyError("invalid type")
-
-        self.datatype = datatype
-
-    # Parsers
-    def parse_float64(self, char *text):
-        cdef int n
-        cdef double ts
-        # Return doubles even in float32 case, since they're going into
-        # a Python array which would upconvert to double anyway.
-        result = []
-        cdef char *end
-        ts = libc.stdlib.strtod(text, &end)
-        if end == text:
-            raise ValueError("bad timestamp")
-        result.append(ts)
-        for n in range(self.count):
-            text = end
-            result.append(libc.stdlib.strtod(text, &end))
-            if end == text:
-                raise ValueError("wrong number of values")
-        n = 0
-        while end[n] == ' ':
-            n += 1
-        if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
-            raise ValueError("extra data on line")
-        return (ts, result)
-
-    def parse_uint16(self, char *text):
-        cdef int n
-        cdef double ts
-        cdef int v
-        result = []
-        cdef char *end
-        ts = libc.stdlib.strtod(text, &end)
-        if end == text:
-            raise ValueError("bad timestamp")
-        result.append(ts)
-        for n in range(self.count):
-            text = end
-            v = libc.stdlib.strtol(text, &end, 10)
-            if v < 0 or v > 65535:
-                raise ValueError("value out of range")
-            result.append(v)
-            if end == text:
-                raise ValueError("wrong number of values")
-        n = 0
-        while end[n] == ' ':
-            n += 1
-        if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
-            raise ValueError("extra data on line")
-        return (ts, result)
-
-    # Formatters
-    def format_float64(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %f" % d[i+1]
-        return s + "\n"
-
-    def format_uint16(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %d" % d[i+1]
-        return s + "\n"
-
-# Get a layout by name
-def get_named(typestring):
-    try:
-        return Layout(typestring)
-    except KeyError:
-        compat = { "PrepData": "float32_8",
-                   "RawData": "uint16_6",
-                   "RawNotchedData": "uint16_9" }
-        return Layout(compat[typestring])
-
-class Parser(object):
-    """Object that parses and stores ASCII data for inclusion into the
-    database"""
-
-    def __init__(self, layout):
-        if issubclass(layout.__class__, Layout):
-            self.layout = layout
-        else:
-            try:
-                self.layout = get_named(layout)
-            except KeyError:
-                raise TypeError("unknown layout")
-
-        self.data = []
-        self.min_timestamp = None
-        self.max_timestamp = None
-
-    def parse(self, textdata):
-        """
-        Parse the data, provided as lines of text, using the current
-        layout, into an internal data structure suitable for a
-        pytables 'table.append(parser.data)'.
-        """
-        cdef double last_ts = 0, ts
-        cdef int n = 0, i
-        cdef char *line
-
-        indata = cStringIO.StringIO(textdata)
-        # Assume any parsing error is a real error.
-        # In the future we might want to skip completely empty lines,
-        # or partial lines right before EOF?
-        try:
-            self.data = []
-            for pyline in indata:
-                line = pyline
-                n += 1
-                if line[0] == '\#':
-                    continue
-                (ts, row) = self.layout.parse(line)
-                if ts <= last_ts:
-                    raise ValueError("timestamp is not "
-                                     "monotonically increasing")
-                last_ts = ts
-                self.data.append(row)
-        except (ValueError, IndexError, TypeError) as e:
-            raise ParserError(n, "error: " + e.message)
-
-        # Mark timestamp ranges
-        if len(self.data):
-            self.min_timestamp = self.data[0][0]
-            self.max_timestamp = self.data[-1][0]
-
-class Formatter(object):
-    """Object that formats database data into ASCII"""
-
-    def __init__(self, layout):
-        if issubclass(layout.__class__, Layout):
-            self.layout = layout
-        else:
-            try:
-                self.layout = get_named(layout)
-            except KeyError:
-                raise TypeError("unknown layout")
-
-    def format(self, data):
-        """
-        Format raw data from the database, using the current layout,
-        as lines of ACSII text.
-        """
-        text = cStringIO.StringIO()
-        try:
-            for row in data:
-                text.write(self.layout.format(row))
-        except (ValueError, IndexError, TypeError) as e:
-            raise FormatterError("formatting error: " + e.message)
-        return text.getvalue()

nilmdb/server/nilmdb.py

@@ -10,16 +10,17 @@ Manages both the SQL database and the table storage backend.
 # Need absolute_import so that "import nilmdb" won't pull in
 # nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
-import nilmdb
+import nilmdb.utils
 from nilmdb.utils.printf import *
-from nilmdb.server.interval import (Interval, DBInterval,
+from nilmdb.utils.time import timestamp_to_string
-                                    IntervalSet, IntervalError)
+
+from nilmdb.utils.interval import IntervalError
+from nilmdb.server.interval import Interval, DBInterval, IntervalSet
+
 from nilmdb.server import bulkdata
-from nilmdb.server.errors import *
+from nilmdb.server.errors import NilmDBError, StreamError, OverlapError
 
 import sqlite3
-import time
-import sys
 import os
 import errno
 import bisect
@@ -33,16 +34,14 @@ import bisect
 #    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
 # 3: at the end of an explicit transaction, e.g. "with self.con as con:"
 #
-# To speed up testing, or if this transaction speed becomes an issue,
+# To speed things up, we can set 'PRAGMA synchronous=OFF'.  Or, it
-# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF.
+# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL'
-
+# give an equivalent speedup more safely.  That is what is used here.
-
-# Don't touch old entries -- just add new ones.
 _sql_schema_updates = {
-    0: """
+    0: { "next": 1, "sql": """
     -- All streams
     CREATE TABLE streams(
-    	id INTEGER PRIMARY KEY,		-- stream ID
+        id INTEGER PRIMARY KEY,		-- stream ID
         path TEXT UNIQUE NOT NULL,	-- path, e.g. '/newton/prep'
         layout TEXT NOT NULL		-- layout name, e.g. float32_8
     );
@@ -63,24 +62,43 @@ _sql_schema_updates = {
         end_pos INTEGER NOT NULL
     );
     CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time);
-    """,
+    """ },
 
-    1: """
+    1: { "next": 3, "sql": """
     -- Generic dictionary-type metadata that can be associated with a stream
     CREATE TABLE metadata(
     	stream_id INTEGER NOT NULL,
         key TEXT NOT NULL,
         value TEXT
     );
-    """,
+    """ },
+
+    2: { "error": "old format with floating-point timestamps requires "
+         "nilmdb 1.3.1 or older" },
+
+    3: { "next": None },
 }
 
 @nilmdb.utils.must_close()
 class NilmDB(object):
     verbose = 0
 
-    def __init__(self, basepath, sync=True, max_results=None,
+    def __init__(self, basepath, max_results=None,
-                 bulkdata_args={}):
+                 max_removals=None, bulkdata_args=None):
+        """Initialize NilmDB at the given basepath.
+        Other arguments are for debugging / testing:
+
+        'max_results' is the max rows to send in a single
+        stream_intervals or stream_extract response.
+
+        'max_removals' is the max rows to delete at once
+        in stream_move.
+
+        'bulkdata_args' is kwargs for the bulkdata module.
+        """
+        if bulkdata_args is None:
+            bulkdata_args = {}
+
         # set up path
         self.basepath = os.path.abspath(basepath)
 
@@ -88,7 +106,9 @@ class NilmDB(object):
         try:
             os.makedirs(self.basepath)
         except OSError as e:
-            if e.errno != errno.EEXIST:
+            if e.errno != errno.EEXIST: # pragma: no cover
+                # (no coverage, because it's hard to trigger this case
+                # if tests are run as root)
                 raise IOError("can't create tree " + self.basepath)
 
         # Our data goes inside it
@@ -96,26 +116,23 @@ class NilmDB(object):
 
         # SQLite database too
         sqlfilename = os.path.join(self.basepath, "data.sql")
-        # We use check_same_thread = False, assuming that the rest
+        self.con = sqlite3.connect(sqlfilename, check_same_thread = True)
-        # of the code (e.g. Server) will be smart and not access this
+        try:
-        # database from multiple threads simultaneously.  Otherwise
+            self._sql_schema_update()
-        # false positives will occur when the database is only opened
+        except Exception: # pragma: no cover
-        # in one thread, and only accessed in another.
+            self.data.close()
-        self.con = sqlite3.connect(sqlfilename, check_same_thread = False)
+            raise
-        self._sql_schema_update()
 
         # See big comment at top about the performance implications of this
-        if sync:
+        self.con.execute("PRAGMA synchronous=NORMAL")
-            self.con.execute("PRAGMA synchronous=FULL")
+        self.con.execute("PRAGMA journal_mode=WAL")
-        else:
-            self.con.execute("PRAGMA synchronous=OFF")
 
         # Approximate largest number of elements that we want to send
-        # in a single reply (for stream_intervals, stream_extract)
+        # in a single reply (for stream_intervals, stream_extract).
-        if max_results:
+        self.max_results = max_results or 16384
-            self.max_results = max_results
+
-        else:
+        # Remove up to this many rows per call to stream_remove.
-            self.max_results = 16384
+        self.max_removals = max_removals or 1048576
 
     def get_basepath(self):
         return self.basepath
@@ -131,16 +148,34 @@ class NilmDB(object):
         version = cur.execute("PRAGMA user_version").fetchone()[0]
         oldversion = version
 
-        while version in _sql_schema_updates:
+        while True:
-            cur.executescript(_sql_schema_updates[version])
+            if version not in _sql_schema_updates: # pragma: no cover
-            version = version + 1
+                raise Exception(self.basepath + ": unknown database version "
+                                + str(version))
+            update = _sql_schema_updates[version]
+            if "error" in update: # pragma: no cover
+                raise Exception(self.basepath + ": can't use database version "
+                                + str(version) + ": " + update["error"])
+            if update["next"] is None:
+                break
+            cur.executescript(update["sql"])
+            version = update["next"]
             if self.verbose: # pragma: no cover
-                printf("Schema updated to %d\n", version)
+                printf("Database schema updated to %d\n", version)
 
         if version != oldversion:
             with self.con:
                 cur.execute("PRAGMA user_version = {v:d}".format(v=version))
 
+    def _check_user_times(self, start, end):
+        if start is None:
+            start = nilmdb.utils.time.min_timestamp
+        if end is None:
+            end = nilmdb.utils.time.max_timestamp
+        if start >= end:
+            raise NilmDBError("start must precede end")
+        return (start, end)
+
     @nilmdb.utils.lru_cache(size = 16)
     def _get_intervals(self, stream_id):
         """
@@ -156,7 +191,7 @@ class NilmDB(object):
                 iset += DBInterval(start_time, end_time,
                                    start_time, end_time,
                                    start_pos, end_pos)
-        except IntervalError as e: # pragma: no cover
+        except IntervalError: # pragma: no cover
             raise NilmDBError("unexpected overlap in ranges table!")
 
         return iset
@@ -264,53 +299,86 @@ class NilmDB(object):
 
         return
 
-    def stream_list(self, path = None, layout = None):
+    def stream_list(self, path = None, layout = None, extended = False):
-        """Return list of [path, layout] lists of all streams
+        """Return list of lists of all streams in the database.
-        in the database.
 
         If path is specified, include only streams with a path that
         matches the given string.
 
         If layout is specified, include only streams with a layout
         that matches the given string.
+
+        If extended = False, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extended = True, returns a list of lists containing
+        more information:
+           path
+           layout
+           interval_min (earliest interval start)
+           interval_max (latest interval end)
+           rows         (total number of rows of data)
+           time         (total time covered by this stream, in timestamp units)
         """
-        where = "WHERE 1=1"
         params = ()
-        if layout:
+        query = "SELECT streams.path, streams.layout"
-            where += " AND layout=?"
+        if extended:
+            query += ", min(ranges.start_time), max(ranges.end_time) "
+            query += ", coalesce(sum(ranges.end_pos - ranges.start_pos), 0) "
+            query += ", coalesce(sum(ranges.end_time - ranges.start_time), 0) "
+        query += " FROM streams"
+        if extended:
+            query += " LEFT JOIN ranges ON streams.id = ranges.stream_id"
+        query += " WHERE 1=1"
+        if layout is not None:
+            query += " AND streams.layout=?"
             params += (layout,)
-        if path:
+        if path is not None:
-            where += " AND path=?"
+            query += " AND streams.path=?"
             params += (path,)
-        result = self.con.execute("SELECT path, layout "
+        query += " GROUP BY streams.id ORDER BY streams.path"
-                                  "FROM streams " + where, params).fetchall()
+        result = self.con.execute(query, params).fetchall()
+        return [ list(x) for x in result ]
 
-        return sorted(list(x) for x in result)
+    def stream_intervals(self, path, start = None, end = None, diffpath = None):
-
-    def stream_intervals(self, path, start = None, end = None):
         """
+        List all intervals in 'path' between 'start' and 'end'.  If
+        'diffpath' is not none, list instead the set-difference
+        between the intervals in the two streams; i.e. all interval
+        ranges that are present in 'path' but not 'diffpath'.
+
         Returns (intervals, restart) tuple.
 
-        intervals is a list of [start,end] timestamps of all intervals
+        'intervals' is a list of [start,end] timestamps of all intervals
         that exist for path, between start and end.
 
-        restart, if nonzero, means that there were too many results to
+        'restart', if not None, means that there were too many results
-        return in a single request.  The data is complete from the
+        to return in a single request.  The data is complete from the
-        starting timestamp to the point at which it was truncated,
+        starting timestamp to the point at which it was truncated, and
-        and a new request with a start time of 'restart' will fetch
+        a new request with a start time of 'restart' will fetch the
-        the next block of data.
+        next block of data.
         """
         stream_id = self._stream_id(path)
         intervals = self._get_intervals(stream_id)
-        requested = Interval(start or 0, end or 1e12)
+        if diffpath:
+            diffstream_id = self._stream_id(diffpath)
+            diffintervals = self._get_intervals(diffstream_id)
+        (start, end) = self._check_user_times(start, end)
+        requested = Interval(start, end)
         result = []
-        for n, i in enumerate(intervals.intersection(requested)):
+        if diffpath:
+            getter = nilmdb.utils.interval.set_difference(
+                intervals.intersection(requested),
+                diffintervals.intersection(requested))
+        else:
+            getter = intervals.intersection(requested)
+        for n, i in enumerate(getter):
             if n >= self.max_results:
                 restart = i.start
                 break
             result.append([i.start, i.end])
         else:
-            restart = 0
+            restart = None
         return (result, restart)
 
     def stream_create(self, path, layout_name):
@@ -373,30 +441,50 @@ class NilmDB(object):
         data.update(newdata)
         self.stream_set_metadata(path, data)
 
+    def stream_rename(self, oldpath, newpath):
+        """Rename a stream."""
+        stream_id = self._stream_id(oldpath)
+
+        # Rename the data
+        self.data.rename(oldpath, newpath)
+
+        # Rename the stream in the database
+        with self.con as con:
+            con.execute("UPDATE streams SET path=? WHERE id=?",
+                        (newpath, stream_id))
+
     def stream_destroy(self, path):
-        """Fully remove a table and all of its data from the database.
+        """Fully remove a table from the database.  Fails if there are
-        No way to undo it!  Metadata is removed."""
+        any intervals data present; remove them first.  Metadata is
+        also removed."""
         stream_id = self._stream_id(path)
 
-        # Delete the cached interval data (if it was cached)
+        # Verify that no intervals are present, and clear the cache
+        iset = self._get_intervals(stream_id)
+        if len(iset):
+            raise NilmDBError("all intervals must be removed before "
+                              "destroying a stream")
         self._get_intervals.cache_remove(self, stream_id)
 
-        # Delete the data
+        # Delete the bulkdata storage
         self.data.destroy(path)
 
-        # Delete metadata, stream, intervals
+        # Delete metadata, stream, intervals (should be none)
         with self.con as con:
             con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
             con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
             con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
 
-    def stream_insert(self, path, start, end, data):
+    def stream_insert(self, path, start, end, data, binary = False):
         """Insert new data into the database.
            path: Path at which to add the data
            start: Starting timestamp
            end: Ending timestamp
-           data: Rows of data, to be passed to PyTable's table.append
+           data: Textual data, formatted according to the layout of path
-                 method.  E.g. nilmdb.layout.Parser.data
+
+           'binary', if True, means that 'data' is raw binary:
+           little-endian, matching the current table's layout,
+           including the int64 timestamp.
            """
         # First check for basic overlap using timestamp info given.
         stream_id = self._stream_id(path)
@@ -406,17 +494,18 @@ class NilmDB(object):
             raise OverlapError("new data overlaps existing data at range: "
                                + str(iset & interval))
 
-        # Insert the data
+        # Tenatively append the data.  This will raise a ValueError if
+        # there are any parse errors.
         table = self.data.getnode(path)
         row_start = table.nrows
-        table.append(data)
+        table.append_data(data, start, end, binary)
         row_end = table.nrows
 
         # Insert the record into the sql database.
         self._add_interval(stream_id, interval, row_start, row_end)
 
         # And that's all
-        return "ok"
+        return
 
     def _find_start(self, table, dbinterval):
         """
@@ -428,7 +517,7 @@ class NilmDB(object):
         # Optimization for the common case where an interval wasn't truncated
         if dbinterval.start == dbinterval.db_start:
             return dbinterval.db_startpos
-        return bisect.bisect_left(bulkdata.TimestampOnlyTable(table),
+        return bisect.bisect_left(table,
                                   dbinterval.start,
                                   dbinterval.db_startpos,
                                   dbinterval.db_endpos)
@@ -447,38 +536,48 @@ class NilmDB(object):
         # want to include the given timestamp in the results.  This is
         # so a queries like 1:00 -> 2:00 and 2:00 -> 3:00 return
         # non-overlapping data.
-        return bisect.bisect_left(bulkdata.TimestampOnlyTable(table),
+        return bisect.bisect_left(table,
                                   dbinterval.end,
                                   dbinterval.db_startpos,
                                   dbinterval.db_endpos)
 
-    def stream_extract(self, path, start = None, end = None, count = False):
+    def stream_extract(self, path, start = None, end = None,
+                       count = False, markup = False, binary = False):
         """
         Returns (data, restart) tuple.
 
-        data is a list of raw data from the database, suitable for
+        'data' is ASCII-formatted data from the database, formatted
-        passing to e.g. nilmdb.layout.Formatter to translate into
+        according to the layout of the stream.
-        textual form.
 
-        restart, if nonzero, means that there were too many results to
+        'restart', if not None, means that there were too many results to
         return in a single request.  The data is complete from the
         starting timestamp to the point at which it was truncated,
         and a new request with a start time of 'restart' will fetch
         the next block of data.
 
-        count, if true, means to not return raw data, but just the count
+        'count', if true, means to not return raw data, but just the count
         of rows that would have been returned.  This is much faster
         than actually fetching the data.  It is not limited by
         max_results.
+
+        'markup', if true, indicates that returned data should be
+        marked with a comment denoting when a particular interval
+        starts, and another comment when an interval ends.
+
+        'binary', if true, means to return raw binary rather than
+        ASCII-formatted data.
         """
         stream_id = self._stream_id(path)
         table = self.data.getnode(path)
         intervals = self._get_intervals(stream_id)
-        requested = Interval(start or 0, end or 1e12)
+        (start, end) = self._check_user_times(start, end)
+        requested = Interval(start, end)
         result = []
         matched = 0
         remaining = self.max_results
-        restart = 0
+        restart = None
+        if binary and (markup or count):
+            raise NilmDBError("binary mode can't be used with markup or count")
         for interval in intervals.intersection(requested):
             # Reading single rows from the table is too slow, so
             # we use two bisections to find both the starting and
@@ -495,36 +594,56 @@ class NilmDB(object):
             row_max = row_start + remaining
             if row_max < row_end:
                 row_end = row_max
-                restart = table[row_max][0]
+                restart = table[row_max]
+
+            # Add markup
+            if markup:
+                result.append("# interval-start " +
+                              timestamp_to_string(interval.start) + "\n")
 
             # Gather these results up
-            result.extend(table[row_start:row_end])
+            result.append(table.get_data(row_start, row_end, binary))
 
             # Count them
             remaining -= row_end - row_start
 
-            if restart:
+            # Add markup, and exit if restart is set.
+            if restart is not None:
+                if markup:
+                    result.append("# interval-end " +
+                                  timestamp_to_string(restart) + "\n")
                 break
+            if markup:
+                result.append("# interval-end " +
+                              timestamp_to_string(interval.end) + "\n")
 
         if count:
             return matched
-        return (result, restart)
+        return ("".join(result), restart)
 
     def stream_remove(self, path, start = None, end = None):
         """
         Remove data from the specified time interval within a stream.
-        Removes all data in the interval [start, end), and intervals
+
-        are truncated or split appropriately.  Returns the number of
+        Removes data in the interval [start, end), and intervals are
-        data points removed.
+        truncated or split appropriately.
+
+        Returns a (removed, restart) tuple.
+
+        'removed' is the number of data points that were removed.
+
+        'restart', if not None, means there were too many rows to
+        remove in a single request.  This function should be called
+        again with a start time of 'restart' to complete the removal.
         """
         stream_id = self._stream_id(path)
         table = self.data.getnode(path)
         intervals = self._get_intervals(stream_id)
-        to_remove = Interval(start or 0, end or 1e12)
+        (start, end) = self._check_user_times(start, end)
+        to_remove = Interval(start, end)
         removed = 0
-
+        remaining = self.max_removals
-        if start == end:
+        restart = None
-            return 0
 
         # Can't remove intervals from within the iterator, so we need to
         # remember what's currently in the intersection now.
@@ -535,6 +654,13 @@ class NilmDB(object):
             row_start = self._find_start(table, dbint)
             row_end = self._find_end(table, dbint)
 
+            # Shorten it if we'll hit the maximum number of removals
+            row_max = row_start + remaining
+            if row_max < row_end:
+                row_end = row_max
+                dbint.end = table[row_max]
+                restart = dbint.end
+
             # Adjust the DBInterval to match the newly found ends
             dbint.db_start = dbint.start
             dbint.db_end = dbint.end
@@ -550,4 +676,7 @@ class NilmDB(object):
             # Count how many were removed
             removed += row_end - row_start
 
-        return removed
+            if restart is not None:
+                break
+
+        return (removed, restart)

nilmdb/server/rocket.c

@@ -0,0 +1,791 @@
+#include <Python.h>
+#include <structmember.h>
+#include <endian.h>
+
+#include <ctype.h>
+#include <stdint.h>
+
+/* Values missing from stdint.h */
+#define UINT8_MIN 0
+#define UINT16_MIN 0
+#define UINT32_MIN 0
+#define UINT64_MIN 0
+
+/* Marker values (if min == max, skip range check) */
+#define FLOAT32_MIN 0
+#define FLOAT32_MAX 0
+#define FLOAT64_MIN 0
+#define FLOAT64_MAX 0
+
+typedef int64_t timestamp_t;
+
+/* This code probably needs to be double-checked for the case where
+   sizeof(long) != 8, so enforce that here with something that will
+   fail at build time.  We assume that the python integer type can
+   hold an int64_t. */
+const static char __long_ok[1 - 2*!(sizeof(int64_t) ==
+				    sizeof(long int))] = { 0 };
+
+/* Somewhat arbitrary, just so we can use fixed sizes for strings
+   etc. */
+static const int MAX_LAYOUT_COUNT = 128;
+
+/* Error object and constants */
+static PyObject *ParseError;
+typedef enum {
+	ERR_OTHER,
+	ERR_NON_MONOTONIC,
+	ERR_OUT_OF_INTERVAL,
+} parseerror_code_t;
+static void add_parseerror_codes(PyObject *module)
+{
+	PyModule_AddIntMacro(module, ERR_OTHER);
+	PyModule_AddIntMacro(module, ERR_NON_MONOTONIC);
+	PyModule_AddIntMacro(module, ERR_OUT_OF_INTERVAL);
+}
+
+/* Helpers to raise ParseErrors.  Use "return raise_str(...)" etc. */
+static PyObject *raise_str(int line, int col, int code, const char *string)
+{
+	PyObject *o;
+	o = Py_BuildValue("(iiis)", line, col, code, string);
+	if (o != NULL) {
+		PyErr_SetObject(ParseError, o);
+		Py_DECREF(o);
+	}
+	return NULL;
+}
+static PyObject *raise_int(int line, int col, int code, int64_t num)
+{
+	PyObject *o;
+	o = Py_BuildValue("(iiil)", line, col, code, num);
+	if (o != NULL) {
+		PyErr_SetObject(ParseError, o);
+		Py_DECREF(o);
+	}
+	return NULL;
+}
+
+/****
+ * Layout and type helpers
+ */
+typedef union {
+	int8_t i;
+	uint8_t u;
+} union8_t;
+typedef union {
+	int16_t i;
+	uint16_t u;
+} union16_t;
+typedef union {
+	int32_t i;
+	uint32_t u;
+	float f;
+} union32_t;
+typedef union {
+	int64_t i;
+	uint64_t u;
+	double d;
+} union64_t;
+
+typedef enum {
+	LAYOUT_TYPE_NONE,
+	LAYOUT_TYPE_INT8,
+	LAYOUT_TYPE_UINT8,
+	LAYOUT_TYPE_INT16,
+	LAYOUT_TYPE_UINT16,
+	LAYOUT_TYPE_INT32,
+	LAYOUT_TYPE_UINT32,
+	LAYOUT_TYPE_INT64,
+	LAYOUT_TYPE_UINT64,
+	LAYOUT_TYPE_FLOAT32,
+	LAYOUT_TYPE_FLOAT64,
+} layout_type_t;
+
+struct {
+	char *string;
+	layout_type_t layout;
+	int size;
+} type_lookup[] = {
+	{ "int8",    LAYOUT_TYPE_INT8,    1 },
+	{ "uint8",   LAYOUT_TYPE_UINT8,   1 },
+	{ "int16",   LAYOUT_TYPE_INT16,   2 },
+	{ "uint16",  LAYOUT_TYPE_UINT16,  2 },
+	{ "int32",   LAYOUT_TYPE_INT32,   4 },
+	{ "uint32",  LAYOUT_TYPE_UINT32,  4 },
+	{ "int64",   LAYOUT_TYPE_INT64,   8 },
+	{ "uint64",  LAYOUT_TYPE_UINT64,  8 },
+	{ "float32", LAYOUT_TYPE_FLOAT32, 4 },
+	{ "float64", LAYOUT_TYPE_FLOAT64, 8 },
+	{ NULL }
+};
+
+/****
+ * Object definition, init, etc
+ */
+
+/* Rocket object */
+typedef struct {
+	PyObject_HEAD
+	layout_type_t layout_type;
+	int layout_count;
+	int binary_size;
+	FILE *file;
+	int file_size;
+} Rocket;
+
+/* Dealloc / new */
+static void Rocket_dealloc(Rocket *self)
+{
+	if (self->file) {
+		fprintf(stderr, "rocket: file wasn't closed\n");
+		fclose(self->file);
+		self->file = NULL;
+	}
+	self->ob_type->tp_free((PyObject *)self);
+}
+
+static PyObject *Rocket_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+	Rocket *self;
+
+	self = (Rocket *)type->tp_alloc(type, 0);
+	if (!self)
+		return NULL;
+	self->layout_type = LAYOUT_TYPE_NONE;
+	self->layout_count = 0;
+	self->binary_size = 0;
+	self->file = NULL;
+	self->file_size = -1;
+	return (PyObject *)self;
+}
+
+/* .__init__(layout, file) */
+static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
+{
+	const char *layout, *path;
+	static char *kwlist[] = { "layout", "file", NULL };
+	if (!PyArg_ParseTupleAndKeywords(args, kwds, "sz", kwlist,
+					 &layout, &path))
+		return -1;
+	if (!layout)
+		return -1;
+	if (path) {
+		if ((self->file = fopen(path, "a+b")) == NULL) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return -1;
+		}
+		self->file_size = -1;
+	} else {
+		self->file = NULL;
+	}
+
+	const char *under;
+	char *tmp;
+	under = strchr(layout, '_');
+	if (!under) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"badly formatted string");
+		return -1;
+	}
+	self->layout_count = strtoul(under+1, &tmp, 10);
+	if (self->layout_count < 1 || *tmp != '\0') {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"bad count");
+		return -1;
+	}
+	if (self->layout_count >= MAX_LAYOUT_COUNT) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"count too high");
+		return -1;
+	}
+
+	int i;
+	for (i = 0; type_lookup[i].string; i++)
+		if (strncmp(layout, type_lookup[i].string, under-layout) == 0)
+			break;
+	if (!type_lookup[i].string) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"bad data type");
+		return -1;
+	}
+	self->layout_type = type_lookup[i].layout;
+	self->binary_size = 8 + (type_lookup[i].size * self->layout_count);
+
+	return 0;
+}
+
+/* .close() */
+static PyObject *Rocket_close(Rocket *self)
+{
+	if (self->file) {
+		fclose(self->file);
+		self->file = NULL;
+	}
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+/* .file_size property */
+static PyObject *Rocket_get_file_size(Rocket *self)
+{
+	if (!self->file) {
+		PyErr_SetString(PyExc_AttributeError, "no file");
+		return NULL;
+	}
+	if (self->file_size < 0) {
+		int oldpos;
+		if (((oldpos = ftell(self->file)) < 0) ||
+		    (fseek(self->file, 0, SEEK_END) < 0) ||
+		    ((self->file_size = ftell(self->file)) < 0) ||
+		    (fseek(self->file, oldpos, SEEK_SET) < 0)) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return NULL;
+		}
+	}
+	return PyInt_FromLong(self->file_size);
+}
+
+/****
+ * Append from string
+ */
+static inline long int strtol10(const char *nptr, char **endptr) {
+	return strtol(nptr, endptr, 10);
+}
+static inline long int strtoul10(const char *nptr, char **endptr) {
+	return strtoul(nptr, endptr, 10);
+}
+
+/* .append_string(count, data, offset, linenum, start, end, last_timestamp) */
+static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
+{
+	int count;
+	const char *data;
+	int offset;
+	const char *linestart;
+	int linenum;
+	timestamp_t start;
+	timestamp_t end;
+	timestamp_t last_timestamp;
+
+	int written = 0;
+	char *endptr;
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+	int i;
+
+	/* It would be nice to use 't#' instead of 's' for data,
+	   but we need the null termination for strto*.  If we had
+	   strnto* that took a length, we could use t# and not require
+	   a copy. */
+	if (!PyArg_ParseTuple(args, "isiilll:append_string", &count,
+			      &data, &offset, &linenum,
+			      &start, &end, &last_timestamp))
+		return NULL;
+
+	/* Skip spaces, but don't skip over a newline. */
+#define SKIP_BLANK(buf) do {			\
+	while (isspace(*buf)) {			\
+		if (*buf == '\n')		\
+			break;			\
+		buf++;				\
+	} } while(0)
+
+	const char *buf = &data[offset];
+	while (written < count && *buf)
+	{
+		linestart = buf;
+		linenum++;
+
+		/* Skip leading whitespace and commented lines */
+		SKIP_BLANK(buf);
+		if (*buf == '#') {
+			while (*buf && *buf != '\n')
+				buf++;
+			if (*buf)
+				buf++;
+			continue;
+		}
+
+		/* Extract timestamp */
+		t64.i = strtoll(buf, &endptr, 10);
+		if (endptr == buf || !isspace(*endptr)) {
+			/* Try parsing as a double instead */
+			t64.d = strtod(buf, &endptr);
+			if (endptr == buf)
+				goto bad_timestamp;
+			if (!isspace(*endptr))
+				goto cant_parse_value;
+			t64.i = round(t64.d);
+		}
+		if (t64.i <= last_timestamp)
+			return raise_int(linenum, buf - linestart + 1,
+					 ERR_NON_MONOTONIC, t64.i);
+		last_timestamp = t64.i;
+		if (t64.i < start || t64.i >= end)
+			return raise_int(linenum, buf - linestart + 1,
+					 ERR_OUT_OF_INTERVAL, t64.i);
+		t64.u = le64toh(t64.u);
+		if (fwrite(&t64.u, 8, 1, self->file) != 1)
+			goto err;
+		buf = endptr;
+
+		/* Parse all values in the line */
+		switch (self->layout_type) {
+#define CS(type, parsefunc, parsetype, realtype, disktype, letoh, bytes) \
+		case LAYOUT_TYPE_##type:				\
+			/* parse and write in a loop */			\
+			for (i = 0; i < self->layout_count; i++) {	\
+				/* skip non-newlines */			\
+				SKIP_BLANK(buf);			\
+				if (*buf == '\n')			\
+					goto wrong_number_of_values;	\
+				/* parse number */			\
+				parsetype = parsefunc(buf, &endptr);	\
+				if (*endptr && !isspace(*endptr))	\
+					goto cant_parse_value;		\
+				/* check limits */			\
+				if (type##_MIN != type##_MAX &&		\
+				    (parsetype < type##_MIN ||		\
+				     parsetype > type##_MAX))		\
+					goto value_out_of_range;	\
+				/* convert to disk representation */	\
+				realtype = parsetype;			\
+				disktype = letoh(disktype);		\
+				/* write it */				\
+				if (fwrite(&disktype, bytes,		\
+					   1, self->file) != 1)		\
+					goto err;			\
+				/* advance buf */			\
+				buf = endptr;				\
+			}						\
+			/* Skip trailing whitespace and comments */	\
+			SKIP_BLANK(buf);				\
+			if (*buf == '#')				\
+				while (*buf && *buf != '\n')		\
+					buf++;				\
+			if (*buf == '\n')				\
+				buf++;					\
+			else if (*buf != '\0')				\
+				goto extra_data_on_line;		\
+			break
+
+			CS(INT8,   strtol10,  t64.i, t8.i,  t8.u,         , 1);
+			CS(UINT8,  strtoul10, t64.u, t8.u,  t8.u,         , 1);
+			CS(INT16,  strtol10,  t64.i, t16.i, t16.u, le16toh, 2);
+			CS(UINT16, strtoul10, t64.u, t16.u, t16.u, le16toh, 2);
+			CS(INT32,  strtol10,  t64.i, t32.i, t32.u, le32toh, 4);
+			CS(UINT32, strtoul10, t64.u, t32.u, t32.u, le32toh, 4);
+			CS(INT64,  strtol10,  t64.i, t64.i, t64.u, le64toh, 8);
+			CS(UINT64, strtoul10, t64.u, t64.u, t64.u, le64toh, 8);
+			CS(FLOAT32, strtod,   t64.d, t32.f, t32.u, le32toh, 4);
+			CS(FLOAT64, strtod,   t64.d, t64.d, t64.u, le64toh, 8);
+#undef CS
+		default:
+			PyErr_SetString(PyExc_TypeError, "unknown type");
+			return NULL;
+		}
+
+		/* Done this line */
+		written++;
+	}
+
+	fflush(self->file);
+
+	/* Build return value and return */
+	offset = buf - data;
+	PyObject *o;
+	o = Py_BuildValue("(iili)", written, offset, last_timestamp, linenum);
+	return o;
+err:
+	PyErr_SetFromErrno(PyExc_OSError);
+	return NULL;
+bad_timestamp:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "bad timestamp");
+cant_parse_value:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "can't parse value");
+wrong_number_of_values:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "wrong number of values");
+value_out_of_range:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "value out of range");
+extra_data_on_line:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "extra data on line");
+}
+
+/****
+ * Append from binary data
+ */
+
+/* .append_binary(count, data, offset, linenum, start, end, last_timestamp) */
+static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
+{
+        int count;
+	const uint8_t *data;
+        int data_len;
+        int linenum;
+	int offset;
+	timestamp_t start;
+	timestamp_t end;
+	timestamp_t last_timestamp;
+
+	if (!PyArg_ParseTuple(args, "it#iilll:append_binary",
+                              &count, &data, &data_len, &offset,
+                              &linenum, &start, &end, &last_timestamp))
+		return NULL;
+
+        /* Advance to offset */
+        if (offset > data_len)
+                return raise_str(0, 0, ERR_OTHER, "bad offset");
+        data += offset;
+        data_len -= offset;
+
+        /* Figure out max number of rows to insert */
+        int rows = data_len / self->binary_size;
+        if (rows > count)
+                rows = count;
+
+        /* Check timestamps */
+        timestamp_t ts;
+	int i;
+        for (i = 0; i < rows; i++) {
+                /* Read raw timestamp, byteswap if needed */
+                memcpy(&ts, &data[i * self->binary_size], 8);
+                ts = le64toh(ts);
+
+                /* Check limits */
+                if (ts <= last_timestamp)
+                        return raise_int(i, 0, ERR_NON_MONOTONIC, ts);
+                last_timestamp = ts;
+                if (ts < start || ts >= end)
+                        return raise_int(i, 0, ERR_OUT_OF_INTERVAL, ts);
+        }
+
+        /* Write binary data */
+        if (fwrite(data, data_len, 1, self->file) != 1) {
+                PyErr_SetFromErrno(PyExc_OSError);
+                return NULL;
+        }
+	fflush(self->file);
+
+	/* Build return value and return */
+	PyObject *o;
+	o = Py_BuildValue("(iili)", rows, offset + rows * self->binary_size,
+                          last_timestamp, linenum);
+	return o;
+}
+
+/****
+ * Extract to string
+ */
+
+static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
+{
+	long count;
+	long offset;
+
+	if (!PyArg_ParseTuple(args, "ll", &offset, &count))
+		return NULL;
+	if (!self->file) {
+		PyErr_SetString(PyExc_Exception, "no file");
+		return NULL;
+	}
+	/* Seek to target location */
+	if (fseek(self->file, offset, SEEK_SET) < 0) {
+		PyErr_SetFromErrno(PyExc_OSError);
+		return NULL;
+	}
+
+	char *str = NULL, *new;
+	long len_alloc = 0;
+	long len = 0;
+	int ret;
+
+	/* min space free in string (and the maximum length of one
+	   line); this is generous */
+	const int min_free = 32 * MAX_LAYOUT_COUNT;
+
+	/* how much to allocate at once */
+	const int alloc_size = 1048576;
+
+	int row, i;
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+	for (row = 0; row < count; row++) {
+		/* Make sure there's space for a line */
+		if ((len_alloc - len) < min_free) {
+			/* grow by 1 meg at a time */
+			len_alloc += alloc_size;
+			new = realloc(str, len_alloc);
+			if (new == NULL)
+				goto err;
+			str = new;
+		}
+
+		/* Read and print timestamp */
+		if (fread(&t64.u, 8, 1, self->file) != 1)
+			goto err;
+		t64.u = le64toh(t64.u);
+		ret = sprintf(&str[len], "%ld", t64.i);
+		if (ret <= 0)
+			goto err;
+		len += ret;
+
+		/* Read and print values */
+		switch (self->layout_type) {
+#define CASE(type, fmt, fmttype, disktype, letoh, bytes)		\
+		case LAYOUT_TYPE_##type:				\
+			/* read and format in a loop */			\
+			for (i = 0; i < self->layout_count; i++) {	\
+				if (fread(&disktype, bytes,		\
+					  1, self->file) != 1)		\
+					goto err;			\
+				disktype = letoh(disktype);		\
+				ret = sprintf(&str[len], " " fmt,	\
+					      fmttype);			\
+				if (ret <= 0)				\
+					goto err;			\
+				len += ret;				\
+			}						\
+			break
+			CASE(INT8,   "%hhd",   t8.i,  t8.u,         , 1);
+			CASE(UINT8,  "%hhu",   t8.u,  t8.u,         , 1);
+			CASE(INT16,  "%hd",    t16.i, t16.u, le16toh, 2);
+			CASE(UINT16, "%hu",    t16.u, t16.u, le16toh, 2);
+			CASE(INT32,  "%d",     t32.i, t32.u, le32toh, 4);
+			CASE(UINT32, "%u",     t32.u, t32.u, le32toh, 4);
+			CASE(INT64,  "%ld",    t64.i, t64.u, le64toh, 8);
+			CASE(UINT64, "%lu",    t64.u, t64.u, le64toh, 8);
+			/* These next two are a bit debatable.  floats
+			   are 6-9 significant figures, so we print 7.
+			   Doubles are 15-19, so we print 17.  This is
+			   similar to the old prep format for float32.
+			*/
+			CASE(FLOAT32, "%.6e",  t32.f, t32.u, le32toh, 4);
+			CASE(FLOAT64, "%.16e", t64.d, t64.u, le64toh, 8);
+#undef CASE
+		default:
+			PyErr_SetString(PyExc_TypeError, "unknown type");
+			if (str) free(str);
+			return NULL;
+		}
+		str[len++] = '\n';
+	}
+
+	PyObject *pystr = PyString_FromStringAndSize(str, len);
+	free(str);
+	return pystr;
+err:
+	if (str) free(str);
+	PyErr_SetFromErrno(PyExc_OSError);
+	return NULL;
+}
+
+/****
+ * Extract to binary string containing raw little-endian binary data
+ */
+static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args)
+{
+	long count;
+	long offset;
+
+	if (!PyArg_ParseTuple(args, "ll", &offset, &count))
+		return NULL;
+	if (!self->file) {
+		PyErr_SetString(PyExc_Exception, "no file");
+		return NULL;
+	}
+	/* Seek to target location */
+	if (fseek(self->file, offset, SEEK_SET) < 0) {
+		PyErr_SetFromErrno(PyExc_OSError);
+		return NULL;
+	}
+
+        uint8_t *str;
+        int len = count * self->binary_size;
+        str = malloc(len);
+        if (str == NULL) {
+                PyErr_SetFromErrno(PyExc_OSError);
+                return NULL;
+        }
+
+        /* Data in the file is already in the desired little-endian
+           binary format, so just read it directly. */
+        if (fread(str, self->binary_size, count, self->file) != count) {
+                free(str);
+                PyErr_SetFromErrno(PyExc_OSError);
+                return NULL;
+        }
+
+	PyObject *pystr = PyBytes_FromStringAndSize((char *)str, len);
+	free(str);
+	return pystr;
+}
+
+/****
+ * Extract timestamp
+ */
+static PyObject *Rocket_extract_timestamp(Rocket *self, PyObject *args)
+{
+	long offset;
+	union64_t t64;
+	if (!PyArg_ParseTuple(args, "l", &offset))
+		return NULL;
+	if (!self->file) {
+		PyErr_SetString(PyExc_Exception, "no file");
+		return NULL;
+	}
+
+	/* Seek to target location and read timestamp */
+	if ((fseek(self->file, offset, SEEK_SET) < 0) ||
+	    (fread(&t64.u, 8, 1, self->file) != 1)) {
+		PyErr_SetFromErrno(PyExc_OSError);
+		return NULL;
+	}
+
+	/* Convert and return */
+	t64.u = le64toh(t64.u);
+	return Py_BuildValue("l", t64.i);
+}
+
+/****
+ * Module and type setup
+ */
+
+static PyGetSetDef Rocket_getsetters[] = {
+	{ "file_size", (getter)Rocket_get_file_size, NULL,
+	  "file size in bytes", NULL },
+	{ NULL },
+};
+
+static PyMemberDef Rocket_members[] = {
+	{ "binary_size", T_INT, offsetof(Rocket, binary_size), 0,
+	  "binary size per row" },
+	{ NULL },
+};
+
+static PyMethodDef Rocket_methods[] = {
+	{ "close",
+          (PyCFunction)Rocket_close, METH_NOARGS,
+	  "close(self)\n\n"
+	  "Close file handle" },
+
+	{ "append_string",
+          (PyCFunction)Rocket_append_string, METH_VARARGS,
+	  "append_string(self, count, data, offset, line, start, end, ts)\n\n"
+          "Parse string and append data.\n"
+	  "\n"
+	  "  count: maximum number of rows to add\n"
+          "  data: string data\n"
+          "  offset: byte offset into data to start parsing\n"
+          "  line: current line number of data\n"
+          "  start: starting timestamp for interval\n"
+          "  end: end timestamp for interval\n"
+          "  ts: last timestamp that was previously parsed\n"
+	  "\n"
+	  "Raises ParseError if timestamps are non-monotonic, outside\n"
+	  "the start/end interval etc.\n"
+	  "\n"
+          "On success, return a tuple:\n"
+          "  added_rows: how many rows were added from the file\n"
+          "  data_offset: current offset into the data string\n"
+          "  last_timestamp: last timestamp we parsed\n"
+          "  linenum: current line number" },
+
+	{ "append_binary",
+	  (PyCFunction)Rocket_append_binary, METH_VARARGS,
+	  "append_binary(self, count, data, offset, line, start, end, ts)\n\n"
+          "Append binary data, which must match the data layout.\n"
+	  "\n"
+	  "  count: maximum number of rows to add\n"
+          "  data: binary data\n"
+          "  offset: byte offset into data to start adding\n"
+          "  line: current line number (unused)\n"
+          "  start: starting timestamp for interval\n"
+          "  end: end timestamp for interval\n"
+          "  ts: last timestamp that was previously parsed\n"
+	  "\n"
+	  "Raises ParseError if timestamps are non-monotonic, outside\n"
+	  "the start/end interval etc.\n"
+	  "\n"
+          "On success, return a tuple:\n"
+          "  added_rows: how many rows were added from the file\n"
+          "  data_offset: current offset into the data string\n"
+          "  last_timestamp: last timestamp we parsed\n"
+          "  linenum: current line number (copied from argument)" },
+
+	{ "extract_string",
+          (PyCFunction)Rocket_extract_string, METH_VARARGS,
+	  "extract_string(self, offset, count)\n\n"
+	  "Extract count rows of data from the file at offset offset.\n"
+	  "Return an ascii formatted string according to the layout" },
+
+	{ "extract_binary",
+	  (PyCFunction)Rocket_extract_binary, METH_VARARGS,
+	  "extract_binary(self, offset, count)\n\n"
+	  "Extract count rows of data from the file at offset offset.\n"
+	  "Return a raw binary string of data matching the data layout." },
+
+	{ "extract_timestamp",
+	  (PyCFunction)Rocket_extract_timestamp, METH_VARARGS,
+	  "extract_timestamp(self, offset)\n\n"
+	  "Extract a single timestamp from the file" },
+
+	{ NULL },
+};
+
+static PyTypeObject RocketType = {
+	PyObject_HEAD_INIT(NULL)
+
+	.tp_name	= "rocket.Rocket",
+	.tp_basicsize	= sizeof(Rocket),
+	.tp_flags	= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+
+	.tp_new		= Rocket_new,
+	.tp_dealloc	= (destructor)Rocket_dealloc,
+	.tp_init	= (initproc)Rocket_init,
+	.tp_methods	= Rocket_methods,
+	.tp_members	= Rocket_members,
+	.tp_getset	= Rocket_getsetters,
+
+	.tp_doc		= ("rocket.Rocket(layout, file)\n\n"
+			   "C implementation of the \"rocket\" data parsing\n"
+			   "interface, which translates between the binary\n"
+			   "format on disk and the ASCII or Python list\n"
+			   "format used when communicating with the rest of\n"
+			   "the system.")
+};
+
+static PyMethodDef module_methods[] = {
+	{ NULL },
+};
+
+PyMODINIT_FUNC
+initrocket(void)
+{
+	PyObject *module;
+
+	RocketType.tp_new = PyType_GenericNew;
+	if (PyType_Ready(&RocketType) < 0)
+		return;
+
+	module = Py_InitModule3("rocket", module_methods,
+				"Rocket data parsing and formatting module");
+	Py_INCREF(&RocketType);
+	PyModule_AddObject(module, "Rocket", (PyObject *)&RocketType);
+
+	ParseError = PyErr_NewException("rocket.ParseError", NULL, NULL);
+	Py_INCREF(ParseError);
+	PyModule_AddObject(module, "ParseError", ParseError);
+	add_parseerror_codes(module);
+
+	return;
+}

nilmdb/server/server.py

@@ -3,31 +3,24 @@
 # Need absolute_import so that "import nilmdb" won't pull in
 # nilmdb.py, but will pull the nilmdb module instead.
 from __future__ import absolute_import
-import nilmdb
+import nilmdb.server
 from nilmdb.utils.printf import *
-from nilmdb.server.errors import *
+from nilmdb.server.errors import NilmDBError
+from nilmdb.utils.time import string_to_timestamp
 
 import cherrypy
 import sys
-import time
 import os
+import socket
 import simplejson as json
 import decorator
+import psutil
 import traceback
 
-try:
-    import cherrypy
-    cherrypy.tools.json_out
-except: # pragma: no cover
-    sys.stderr.write("Cherrypy 3.2+ required\n")
-    sys.exit(1)
-
 class NilmApp(object):
     def __init__(self, db):
         self.db = db
 
-version = "1.2"
-
 # Decorators
 def chunked_response(func):
     """Decorator to enable chunked responses."""
@@ -54,12 +47,17 @@ def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
     bug #1200.  This throws them as generic Exceptions instead so that
     they make it through.
     """
+    exc_info = None
     try:
         for val in func(*args, **kwargs):
             yield val
-    except (LookupError, UnicodeError) as e:
+    except (LookupError, UnicodeError):
-        raise Exception("bug workaround; real exception is:\n" +
+        # Re-raise it, but maintain the original traceback
-                        traceback.format_exc())
+        exc_info = sys.exc_info()
+        new_exc = Exception(exc_info[0].__name__ + ": " + str(exc_info[1]))
+        raise new_exc, None, exc_info[2]
+    finally:
+        del exc_info
 
 def exception_to_httperror(*expected):
     """Return a decorator-generating function that catches expected
@@ -70,23 +68,81 @@ def exception_to_httperror(*expected):
             pass
     """
     def wrapper(func, *args, **kwargs):
+        exc_info = None
         try:
             return func(*args, **kwargs)
-        except expected as e:
+        except expected:
-            message = sprintf("%s", str(e))
+            # Re-raise it, but maintain the original traceback
-            raise cherrypy.HTTPError("400 Bad Request", message)
+            exc_info = sys.exc_info()
+            new_exc = cherrypy.HTTPError("400 Bad Request", str(exc_info[1]))
+            raise new_exc, None, exc_info[2]
+        finally:
+            del exc_info
     # We need to preserve the function's argspecs for CherryPy to
     # handle argument errors correctly.  Decorator.decorator takes
     # care of that.
     return decorator.decorator(wrapper)
 
+# Custom CherryPy tools
+
+def CORS_allow(methods):
+    """This does several things:
+
+    Handles CORS preflight requests.
+    Adds Allow: header to all requests.
+    Raise 405 if request.method not in method.
+
+    It is similar to cherrypy.tools.allow, with the CORS stuff added.
+    """
+    request = cherrypy.request.headers
+    response = cherrypy.response.headers
+
+    if not isinstance(methods, (tuple, list)): # pragma: no cover
+        methods = [ methods ]
+    methods = [ m.upper() for m in methods if m ]
+    if not methods: # pragma: no cover
+        methods = [ 'GET', 'HEAD' ]
+    elif 'GET' in methods and 'HEAD' not in methods: # pragma: no cover
+        methods.append('HEAD')
+    response['Allow'] = ', '.join(methods)
+
+    # Allow all origins
+    if 'Origin' in request:
+        response['Access-Control-Allow-Origin'] = request['Origin']
+
+    # If it's a CORS request, send response.
+    request_method = request.get("Access-Control-Request-Method", None)
+    request_headers = request.get("Access-Control-Request-Headers", None)
+    if (cherrypy.request.method == "OPTIONS" and
+        request_method and request_headers):
+        response['Access-Control-Allow-Headers'] = request_headers
+        response['Access-Control-Allow-Methods'] = ', '.join(methods)
+        # Try to stop further processing and return a 200 OK
+        cherrypy.response.status = "200 OK"
+        cherrypy.response.body = ""
+        cherrypy.request.handler = lambda: ""
+        return
+
+    # Reject methods that were not explicitly allowed
+    if cherrypy.request.method not in methods:
+        raise cherrypy.HTTPError(405)
+
+cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
+
+# Helper for json_in tool to process JSON data into normal request
+# parameters.
+def json_to_request_params(body):
+    cherrypy.lib.jsontools.json_processor(body)
+    if not isinstance(cherrypy.request.json, dict):
+        raise cherrypy.HTTPError(415)
+    cherrypy.request.params.update(cherrypy.request.json)
+
 # CherryPy apps
 class Root(NilmApp):
     """Root application for NILM database"""
 
-    def __init__(self, db, version):
+    def __init__(self, db):
         super(Root, self).__init__(db)
-        self.server_version = version
 
     # /
     @cherrypy.expose
@@ -102,38 +158,62 @@ class Root(NilmApp):
     @cherrypy.expose
     @cherrypy.tools.json_out()
     def version(self):
-        return self.server_version
+        return nilmdb.__version__
 
-    # /dbpath
+    # /dbinfo
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def dbpath(self):
+    def dbinfo(self):
-        return self.db.get_basepath()
+        """Return a dictionary with the database path,
-
+        size of the database in bytes, and free disk space in bytes"""
-    # /dbsize
+        path = self.db.get_basepath()
-    @cherrypy.expose
+        return { "path": path,
-    @cherrypy.tools.json_out()
+                 "size": nilmdb.utils.du(path),
-    def dbsize(self):
+                 "free": psutil.disk_usage(path).free }
-        return nilmdb.utils.du(self.db.get_basepath())
 
 class Stream(NilmApp):
     """Stream-specific operations"""
 
+    # Helpers
+    def _get_times(self, start_param, end_param):
+        (start, end) = (None, None)
+        if start_param is not None:
+            start = string_to_timestamp(start_param)
+        if end_param is not None:
+            end = string_to_timestamp(end_param)
+        if start is not None and end is not None:
+            if start >= end:
+                raise cherrypy.HTTPError(
+                    "400 Bad Request",
+                    sprintf("start must precede end (%s >= %s)",
+                            start_param, end_param))
+        return (start, end)
+
     # /stream/list
-    # /stream/list?layout=PrepData
+    # /stream/list?layout=float32_8
-    # /stream/list?path=/newton/prep
+    # /stream/list?path=/newton/prep&extended=1
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def list(self, path = None, layout = None):
+    def list(self, path = None, layout = None, extended = None):
         """List all streams in the database.  With optional path or
         layout parameter, just list streams that match the given path
-        or layout"""
+        or layout.
-        return self.db.stream_list(path, layout)
 
-    # /stream/create?path=/newton/prep&layout=PrepData
+        If extent is not given, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extended is provided, returns a list of lists containing
+        extended info: [ path, layout, extent_min, extent_max,
+        total_rows, total_seconds ].  More data may be added.
+        """
+        return self.db.stream_list(path, layout, bool(extended))
+
+    # /stream/create?path=/newton/prep&layout=float32_8
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def create(self, path, layout):
         """Create a new stream in the database.  Provide path
         and one of the nilmdb.layout.layouts keys.
@@ -142,12 +222,24 @@ class Stream(NilmApp):
 
     # /stream/destroy?path=/newton/prep
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def destroy(self, path):
-        """Delete a stream and its associated data."""
+        """Delete a stream.  Fails if any data is still present."""
         return self.db.stream_destroy(path)
 
+    # /stream/rename?oldpath=/newton/prep&newpath=/newton/prep/1
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    def rename(self, oldpath, newpath):
+        """Rename a stream."""
+        return self.db.stream_rename(oldpath, newpath)
+
     # /stream/get_metadata?path=/newton/prep
     # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
     @cherrypy.expose
@@ -172,147 +264,146 @@ class Stream(NilmApp):
                 result[k] = None
         return result
 
+    # Helper for set_metadata and get_metadata
+    def _metadata_helper(self, function, path, data):
+        if not isinstance(data, dict):
+            try:
+                data = dict(json.loads(data))
+            except TypeError as e:
+                raise NilmDBError("can't parse 'data' parameter: " + e.message)
+        for key in data:
+            if not (isinstance(data[key], basestring) or
+                    isinstance(data[key], float) or
+                    isinstance(data[key], int)):
+                raise NilmDBError("metadata values must be a string or number")
+        function(path, data)
+
     # /stream/set_metadata?path=/newton/prep&data=<json>
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError, TypeError)
+    @exception_to_httperror(NilmDBError, LookupError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def set_metadata(self, path, data):
-        """Set metadata for the named stream, replacing any
+        """Set metadata for the named stream, replacing any existing
-        existing metadata.  Data should be a json-encoded
+        metadata.  Data can be json-encoded or a plain dictionary."""
-        dictionary"""
+        self._metadata_helper(self.db.stream_set_metadata, path, data)
-        data_dict = json.loads(data)
-        self.db.stream_set_metadata(path, data_dict)
-        return "ok"
 
     # /stream/update_metadata?path=/newton/prep&data=<json>
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError, TypeError)
+    @exception_to_httperror(NilmDBError, LookupError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def update_metadata(self, path, data):
-        """Update metadata for the named stream.  Data
+        """Set metadata for the named stream, replacing any existing
-        should be a json-encoded dictionary"""
+        metadata.  Data can be json-encoded or a plain dictionary."""
-        data_dict = json.loads(data)
+        self._metadata_helper(self.db.stream_update_metadata, path, data)
-        self.db.stream_update_metadata(path, data_dict)
-        return "ok"
 
     # /stream/insert?path=/newton/prep
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    #@cherrypy.tools.disable_prb()
+    @exception_to_httperror(NilmDBError, ValueError)
-    def insert(self, path, start, end):
+    @cherrypy.tools.CORS_allow(methods = ["PUT"])
+    def insert(self, path, start, end, binary = False):
         """
         Insert new data into the database.  Provide textual data
         (matching the path's layout) as a HTTP PUT.
+
+        If 'binary' is True, expect raw binary data, rather than lines
+        of ASCII-formatted data.  Raw binary data is always
+        little-endian and matches the database types (including an
+        int64 timestamp).
         """
         # Important that we always read the input before throwing any
         # errors, to keep lengths happy for persistent connections.
-        # However, CherryPy 3.2.2 has a bug where this fails for GET
+        # Note that CherryPy 3.2.2 has a bug where this fails for GET
-        # requests, so catch that. (issue #1134)
+        # requests, if we ever want to handle those (issue #1134)
-        try:
+        body = cherrypy.request.body.read()
-            body = cherrypy.request.body.read()
+
-        except TypeError:
+        # Verify content type for binary data
-            raise cherrypy.HTTPError("400 Bad Request", "No request body")
+        content_type = cherrypy.request.headers.get('content-type')
+        if binary and content_type:
+            if content_type != "application/octet-stream":
+                raise cherrypy.HTTPError("400", "Content type must be "
+                                         "application/octet-stream for "
+                                         "binary data, not " + content_type)
 
         # Check path and get layout
-        streams = self.db.stream_list(path = path)
+        if len(self.db.stream_list(path = path)) != 1:
-        if len(streams) != 1:
+            raise cherrypy.HTTPError("404", "No such stream: " + path)
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
-
-        # Parse the input data
-        try:
-            parser = nilmdb.server.layout.Parser(layout)
-            parser.parse(body)
-        except nilmdb.server.layout.ParserError as e:
-            raise cherrypy.HTTPError("400 Bad Request",
-                                     "error parsing input data: " +
-                                     e.message)
-
-        if (not parser.min_timestamp or not parser.max_timestamp or
-            not len(parser.data)):
-            raise cherrypy.HTTPError("400 Bad Request",
-                                     "no data provided")
 
         # Check limits
-        start = float(start)
+        (start, end) = self._get_times(start, end)
-        end = float(end)
-        if parser.min_timestamp < start:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.min_timestamp) +
-                                     " < start time " + repr(start))
-        if parser.max_timestamp >= end:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.max_timestamp) +
-                                     " >= end time " + repr(end))
 
-        # Now do the nilmdb insert, passing it the parser full of data.
+        # Pass the data directly to nilmdb, which will parse it and
-        try:
+        # raise a ValueError if there are any problems.
-            result = self.db.stream_insert(path, start, end, parser.data)
+        self.db.stream_insert(path, start, end, body, binary)
-        except NilmDBError as e:
-            raise cherrypy.HTTPError("400 Bad Request", e.message)
 
         # Done
-        return "ok"
+        return
 
     # /stream/remove?path=/newton/prep
     # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def remove(self, path, start = None, end = None):
         """
         Remove data from the backend database.  Removes all data in
         the interval [start, end).  Returns the number of data points
         removed.
         """
-        if start is not None:
+        (start, end) = self._get_times(start, end)
-            start = float(start)
+        total_removed = 0
-        if end is not None:
+        while True:
-            end = float(end)
+            (removed, restart) = self.db.stream_remove(path, start, end)
-        if start is not None and end is not None:
+            total_removed += removed
-            if end < start:
+            if restart is None:
-                raise cherrypy.HTTPError("400 Bad Request",
+                break
-                                         "end before start")
+            start = restart
-        return self.db.stream_remove(path, start, end)
+        return total_removed
 
     # /stream/intervals?path=/newton/prep
     # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
+    # /stream/intervals?path=/newton/prep&diffpath=/newton/prep2
     @cherrypy.expose
     @chunked_response
-    @response_type("text/plain")
+    @response_type("application/x-json-stream")
-    def intervals(self, path, start = None, end = None):
+    def intervals(self, path, start = None, end = None, diffpath = None):
         """
         Get intervals from backend database.  Streams the resulting
-        intervals as JSON strings separated by newlines.  This may
+        intervals as JSON strings separated by CR LF pairs.  This may
         make multiple requests to the nilmdb backend to avoid causing
         it to block for too long.
 
-        Note that the response type is set to 'text/plain' even
+        Returns intervals between 'start' and 'end' belonging to
-        though we're sending back JSON; this is because we're not
+        'path'.  If 'diff' is provided, the set-difference between
-        really returning a single JSON object.
+        intervals in 'path' and intervals in 'diffpath' are
+        returned instead.
+
+        Note that the response type is the non-standard
+        'application/x-json-stream' for lack of a better option.
         """
-        if start is not None:
+        (start, end) = self._get_times(start, end)
-            start = float(start)
-        if end is not None:
-            end = float(end)
 
-        if start is not None and end is not None:
+        if len(self.db.stream_list(path = path)) != 1:
-            if end < start:
+            raise cherrypy.HTTPError("404", "No such stream: " + path)
-                raise cherrypy.HTTPError("400 Bad Request",
-                                         "end before start")
 
-        streams = self.db.stream_list(path = path)
+        if diffpath and len(self.db.stream_list(path = diffpath)) != 1:
-        if len(streams) != 1:
+            raise cherrypy.HTTPError("404", "No such stream: " + diffpath)
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
 
         @workaround_cp_bug_1200
         def content(start, end):
             # Note: disable chunked responses to see tracebacks from here.
             while True:
-                (intervals, restart) = self.db.stream_intervals(path,start,end)
+                (ints, restart) = self.db.stream_intervals(path, start, end,
-                response = ''.join([ json.dumps(i) + "\n" for i in intervals ])
+                                                           diffpath)
+                response = ''.join([ json.dumps(i) + "\r\n" for i in ints ])
                 yield response
-                if restart == 0:
+                if restart is None:
                     break
                 start = restart
         return content(start, end)
@@ -320,54 +411,58 @@ class Stream(NilmApp):
     # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
     @cherrypy.expose
     @chunked_response
-    @response_type("text/plain")
+    def extract(self, path, start = None, end = None,
-    def extract(self, path, start = None, end = None, count = False):
+                count = False, markup = False, binary = False):
         """
         Extract data from backend database.  Streams the resulting
         entries as ASCII text lines separated by newlines.  This may
         make multiple requests to the nilmdb backend to avoid causing
         it to block for too long.
 
-        Add count=True to return a count rather than actual data.
+        If 'count' is True, returns a count rather than actual data.
-        """
-        if start is not None:
-            start = float(start)
-        if end is not None:
-            end = float(end)
 
-        # Check parameters
+        If 'markup' is True, adds comments to the stream denoting each
-        if start is not None and end is not None:
+        interval's start and end timestamp.
-            if end < start:
+
-                raise cherrypy.HTTPError("400 Bad Request",
+        If 'binary' is True, return raw binary data, rather than lines
-                                         "end before start")
+        of ASCII-formatted data.  Raw binary data is always
+        little-endian and matches the database types (including an
+        int64 timestamp).
+        """
+        (start, end) = self._get_times(start, end)
 
         # Check path and get layout
-        streams = self.db.stream_list(path = path)
+        if len(self.db.stream_list(path = path)) != 1:
-        if len(streams) != 1:
+            raise cherrypy.HTTPError("404", "No such stream: " + path)
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
 
-        # Get formatter
+        if binary:
-        formatter = nilmdb.server.layout.Formatter(layout)
+            content_type = "application/octet-stream"
+            if markup or count:
+                raise cherrypy.HTTPError("400", "can't mix binary and "
+                                         "markup or count modes")
+        else:
+            content_type = "text/plain"
+        cherrypy.response.headers['Content-Type'] = content_type
 
         @workaround_cp_bug_1200
-        def content(start, end, count):
+        def content(start, end):
             # Note: disable chunked responses to see tracebacks from here.
             if count:
-                matched = self.db.stream_extract(path, start, end, count)
+                matched = self.db.stream_extract(path, start, end,
+                                                 count = True)
                 yield sprintf("%d\n", matched)
                 return
 
             while True:
-                (data, restart) = self.db.stream_extract(path, start, end)
+                (data, restart) = self.db.stream_extract(
+                    path, start, end, count = False,
+                    markup = markup, binary = binary)
+                yield data
 
-                # Format the data and yield it
+                if restart is None:
-                yield formatter.format(data)
-
-                if restart == 0:
                     return
                 start = restart
-        return content(start, end, count)
+        return content(start, end)
 
 class Exiter(object):
     """App that exits the server, for testing"""
@@ -385,33 +480,54 @@ class Server(object):
                  stoppable = False,       # whether /exit URL exists
                  embedded = True,         # hide diagnostics and output, etc
                  fast_shutdown = False,   # don't wait for clients to disconn.
-                 force_traceback = False  # include traceback in all errors
+                 force_traceback = False, # include traceback in all errors
+                 basepath = '',           # base URL path for cherrypy.tree
                  ):
-        self.version = version
+        # Save server version, just for verification during tests
+        self.version = nilmdb.__version__
 
-        # Need to wrap DB object in a serializer because we'll call
-        # into it from separate threads.
         self.embedded = embedded
-        self.db = nilmdb.utils.Serializer(db)
+        self.db = db
+        if not getattr(db, "_thread_safe", None):
+            raise KeyError("Database object " + str(db) + " doesn't claim "
+                           "to be thread safe.  You should pass "
+                           "nilmdb.utils.serializer_proxy(NilmDB)(args) "
+                           "rather than NilmDB(args).")
+
+        # Build up global server configuration
         cherrypy.config.update({
             'server.socket_host': host,
             'server.socket_port': port,
             'engine.autoreload_on': False,
-            'server.max_request_body_size': 4*1024*1024,
+            'server.max_request_body_size': 8*1024*1024,
-            'error_page.default': self.json_error_page,
             })
         if self.embedded:
             cherrypy.config.update({ 'environment': 'embedded' })
 
-        # Send a permissive Access-Control-Allow-Origin (CORS) header
+        # Build up application specific configuration
-        # with all responses so that browsers can send cross-domain
+        app_config = {}
-        # requests to this server.
+        app_config.update({
-        cherrypy.config.update({ 'response.headers.Access-Control-Allow-Origin':
+            'error_page.default': self.json_error_page,
-                                 '*' })
+            })
+
+        # Some default headers to just help identify that things are working
+        app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
+
+        # Set up Cross-Origin Resource Sharing (CORS) handler so we
+        # can correctly respond to browsers' CORS preflight requests.
+        # This also limits verbs to GET and HEAD by default.
+        app_config.update({ 'tools.CORS_allow.on': True,
+                            'tools.CORS_allow.methods': ['GET', 'HEAD'] })
+
+        # Configure the 'json_in' tool to also allow other content-types
+        # (like x-www-form-urlencoded), and to treat JSON as a dict that
+        # fills requests.param.
+        app_config.update({ 'tools.json_in.force': False,
+                            'tools.json_in.processor': json_to_request_params })
 
         # Send tracebacks in error responses.  They're hidden by the
         # error_page function for client errors (code 400-499).
-        cherrypy.config.update({ 'request.show_tracebacks' : True })
+        app_config.update({ 'request.show_tracebacks' : True })
         self.force_traceback = force_traceback
 
         # Patch CherryPy error handler to never pad out error messages.
@@ -419,11 +535,13 @@ class Server(object):
         # error messages.
         cherrypy._cperror._ie_friendly_error_sizes = {}
 
-        cherrypy.tree.apps = {}
+        # Build up the application and mount it
-        cherrypy.tree.mount(Root(self.db, self.version), "/")
+        root = Root(self.db)
-        cherrypy.tree.mount(Stream(self.db), "/stream")
+        root.stream = Stream(self.db)
         if stoppable:
-            cherrypy.tree.mount(Exiter(), "/exit")
+            root.exit = Exiter()
+        cherrypy.tree.apps = {}
+        cherrypy.tree.mount(root, basepath, config = { "/" : app_config })
 
         # Shutdowns normally wait for clients to disconnect.  To speed
         # up tests, set fast_shutdown = True
@@ -433,6 +551,9 @@ class Server(object):
         else:
             cherrypy.server.shutdown_timeout = 5
 
+        # Set up the WSGI application pointer for external programs
+        self.wsgi_application = cherrypy.tree
+
     def json_error_page(self, status, message, traceback, version):
         """Return a custom error page in JSON so the client can parse it"""
         errordata = { "status" : status,
@@ -444,7 +565,7 @@ class Server(object):
             if not self.force_traceback:
                 if code >= 400 and code <= 499:
                     errordata["traceback"] = ""
-        except Exception as e: # pragma: no cover
+        except Exception: # pragma: no cover
             pass
         # Override the response type, which was previously set to text/html
         cherrypy.serving.response.headers['Content-Type'] = (
@@ -499,3 +620,55 @@ class Server(object):
 
     def stop(self):
         cherrypy.engine.exit()
+
+# Use a single global nilmdb.server.NilmDB and nilmdb.server.Server
+# instance since the database can only be opened once.  For this to
+# work, the web server must use only a single process and single
+# Python interpreter.  Multiple threads are OK.
+_wsgi_server = None
+def wsgi_application(dbpath, basepath): # pragma: no cover
+    """Return a WSGI application object with a database at the
+    specified path.
+
+    'dbpath' is a filesystem location, e.g. /home/nilm/db
+
+    'basepath' is the URL path of the application base, which
+    is the same as the first argument to Apache's WSGIScriptAlias
+    directive.
+    """
+    def application(environ, start_response):
+        global _wsgi_server
+        if _wsgi_server is None:
+            # Try to start the server
+            try:
+                db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(dbpath)
+                _wsgi_server = nilmdb.server.Server(
+                    db, embedded = True,
+                    basepath = basepath.rstrip('/'))
+            except Exception:
+                # Build an error message on failure
+                import pprint
+                err = sprintf("Initializing database at path '%s' failed:\n\n",
+                              dbpath)
+                err += traceback.format_exc()
+                try:
+                    import pwd
+                    import grp
+                    err += sprintf("\nRunning as: uid=%d (%s), gid=%d (%s) "
+                                   "on host %s, pid %d\n",
+                                   os.getuid(), pwd.getpwuid(os.getuid())[0],
+                                   os.getgid(), grp.getgrgid(os.getgid())[0],
+                                   socket.gethostname(), os.getpid())
+                except ImportError:
+                    pass
+                err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
+        if _wsgi_server is None:
+            # Serve up the error with our own mini WSGI app.
+            headers = [ ('Content-type', 'text/plain'),
+                        ('Content-length', str(len(err))) ]
+            start_response("500 Internal Server Error", headers)
+            return [err]
+
+        # Call the normal application
+        return _wsgi_server.wsgi_application(environ, start_response)
+    return application

nilmdb/utils/__init__.py

@@ -1,11 +1,15 @@
 """NilmDB utilities"""
 
-from .timer import Timer
+from __future__ import absolute_import
-from .iteratorizer import Iteratorizer
+from nilmdb.utils.timer import Timer
-from .serializer import Serializer
+from nilmdb.utils.serializer import serializer_proxy
-from .lrucache import lru_cache
+from nilmdb.utils.lrucache import lru_cache
-from .diskusage import du
+from nilmdb.utils.diskusage import du, human_size
-from .mustclose import must_close
+from nilmdb.utils.mustclose import must_close
-from .urllib import urlencode
+from nilmdb.utils import atomic
-from . import misc
+import nilmdb.utils.threadsafety
-from . import atomic
+import nilmdb.utils.fallocate
+import nilmdb.utils.time
+import nilmdb.utils.iterator
+import nilmdb.utils.interval
+import nilmdb.utils.lock

nilmdb/utils/diskusage.py

@@ -1,7 +1,7 @@
 import os
 from math import log
 
-def sizeof_fmt(num):
+def human_size(num):
     """Human friendly file size"""
     unit_list = zip(['bytes', 'kiB', 'MiB', 'GiB', 'TiB'], [0, 0, 1, 2, 2])
     if num > 1:
@@ -15,15 +15,11 @@ def sizeof_fmt(num):
     if num == 1: # pragma: no cover
         return '1 byte'
 
-def du_bytes(path):
+def du(path):
     """Like du -sb, returns total size of path in bytes."""
     size = os.path.getsize(path)
     if os.path.isdir(path):
-        for file in os.listdir(path):
+        for thisfile in os.listdir(path):
-            filepath = os.path.join(path, file)
+            filepath = os.path.join(path, thisfile)
-            size += du_bytes(filepath)
+            size += du(filepath)
     return size
-
-def du(path):
-    """Like du -sh, returns total size of path as a human-readable string."""
-    return sizeof_fmt(du_bytes(path))

nilmdb/utils/fallocate.py

@@ -0,0 +1,49 @@
+# Implementation of hole punching via fallocate, if the OS
+# and filesystem support it.
+
+try:
+    import os
+    import ctypes
+    import ctypes.util
+
+    def make_fallocate():
+        libc_name = ctypes.util.find_library('c')
+        libc = ctypes.CDLL(libc_name, use_errno=True)
+
+        _fallocate = libc.fallocate
+        _fallocate.restype = ctypes.c_int
+        _fallocate.argtypes = [ ctypes.c_int, ctypes.c_int,
+                                ctypes.c_int64, ctypes.c_int64 ]
+
+        del libc
+        del libc_name
+
+        def fallocate(fd, mode, offset, len_):
+            res = _fallocate(fd, mode, offset, len_)
+            if res != 0: # pragma: no cover
+                errno = ctypes.get_errno()
+                raise IOError(errno, os.strerror(errno))
+        return fallocate
+
+    fallocate = make_fallocate()
+    del make_fallocate
+except Exception: # pragma: no cover
+    fallocate = None
+
+FALLOC_FL_KEEP_SIZE = 0x01
+FALLOC_FL_PUNCH_HOLE = 0x02
+
+def punch_hole(filename, offset, length, ignore_errors = True):
+    """Punch a hole in the file.  This isn't well supported, so errors
+    are ignored by default."""
+    try:
+        if fallocate is None: # pragma: no cover
+            raise IOError("fallocate not available")
+        with open(filename, "r+") as f:
+            fallocate(f.fileno(),
+                      FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+                      offset, length)
+    except IOError: # pragma: no cover
+        if ignore_errors:
+            return
+        raise

nilmdb/utils/interval.py

@@ -0,0 +1,106 @@
+"""Interval.  Like nilmdb.server.interval, but re-implemented here
+in plain Python so clients have easier access to it.
+
+Intervals are half-open, ie. they include data points with timestamps
+[start, end)
+"""
+
+import nilmdb.utils.time
+import nilmdb.utils.iterator
+
+class IntervalError(Exception):
+    """Error due to interval overlap, etc"""
+    pass
+
+# Interval
+class Interval:
+    """Represents an interval of time."""
+
+    def __init__(self, start, end):
+        """
+        'start' and 'end' are arbitrary numbers that represent time
+        """
+        if start >= end:
+            # Explicitly disallow zero-width intervals (since they're half-open)
+            raise IntervalError("start %s must precede end %s" % (start, end))
+        self.start = start
+        self.end = end
+
+    def __repr__(self):
+        s = repr(self.start) + ", " + repr(self.end)
+        return self.__class__.__name__ + "(" + s + ")"
+
+    def __str__(self):
+        return ("[" + nilmdb.utils.time.timestamp_to_string(self.start) +
+                " -> " + nilmdb.utils.time.timestamp_to_string(self.end) + ")")
+
+    def __cmp__(self, other):
+        """Compare two intervals.  If non-equal, order by start then end"""
+        return cmp(self.start, other.start) or cmp(self.end, other.end)
+
+    def intersects(self, other):
+        """Return True if two Interval objects intersect"""
+        if not isinstance(other, Interval):
+            raise TypeError("need an Interval")
+        if self.end <= other.start or self.start >= other.end:
+            return False
+        return True
+
+    def subset(self, start, end):
+        """Return a new Interval that is a subset of this one"""
+        # A subclass that tracks additional data might override this.
+        if start < self.start or end > self.end:
+            raise IntervalError("not a subset")
+        return Interval(start, end)
+
+def set_difference(a, b):
+    """
+    Compute the difference (a \\ b) between the intervals in 'a' and
+    the intervals in 'b'; i.e., the ranges that are present in 'self'
+    but not 'other'.
+
+    'a' and 'b' must both be iterables.
+
+    Returns a generator that yields each interval in turn.
+    Output intervals are built as subsets of the intervals in the
+    first argument (a).
+    """
+    # Iterate through all starts and ends in sorted order.  Add a
+    # tag to the iterator so that we can figure out which one they
+    # were, after sorting.
+    def decorate(it, key_start, key_end):
+        for i in it:
+            yield i.start, key_start, i
+            yield i.end, key_end, i
+    a_iter = decorate(iter(a), 0, 2)
+    b_iter = decorate(iter(b), 1, 3)
+
+    # Now iterate over the timestamps of each start and end.
+    # At each point, evaluate which type of end it is, to determine
+    # how to build up the output intervals.
+    a_interval = None
+    b_interval = None
+    out_start = None
+    for (ts, k, i) in nilmdb.utils.iterator.imerge(a_iter, b_iter):
+        if k == 0:
+            # start a interval
+            a_interval = i
+            if b_interval is None:
+                out_start = ts
+        elif k == 1:
+            # start b interval
+            b_interval = i
+            if out_start is not None and out_start != ts:
+                yield a_interval.subset(out_start, ts)
+            out_start = None
+        elif k == 2:
+            # end a interval
+            if out_start is not None and out_start != ts:
+                yield a_interval.subset(out_start, ts)
+            out_start = None
+            a_interval = None
+        elif k == 3:
+            # end b interval
+            b_interval = None
+            if a_interval:
+                out_start = ts

nilmdb/utils/iterator.py

@@ -0,0 +1,36 @@
+# Misc iterator tools
+
+# Iterator merging, based on http://code.activestate.com/recipes/491285/
+import heapq
+def imerge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Equivalent to:  sorted(itertools.chain(*iterables))
+
+    >>> list(imerge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    heappop, siftup, _Stop = heapq.heappop, heapq._siftup, StopIteration
+
+    h = []
+    h_append = h.append
+    for it in map(iter, iterables):
+        try:
+            next = it.next
+            h_append([next(), next])
+        except _Stop:
+            pass
+    heapq.heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, next = s = h[0]      # raises IndexError when h is empty
+                yield v
+                s[0] = next()           # raises StopIteration when exhausted
+                siftup(h, 0)            # restore heap condition
+        except _Stop:
+            heappop(h)                  # remove empty iterator
+        except IndexError:
+            return

nilmdb/utils/iteratorizer.py

@@ -1,99 +0,0 @@
-import Queue
-import threading
-import sys
-import contextlib
-
-# This file provides a context manager that converts a function
-# that takes a callback into a generator that returns an iterable.
-# This is done by running the function in a new thread.
-
-# Based partially on http://stackoverflow.com/questions/9968592/
-
-class IteratorizerThread(threading.Thread):
-    def __init__(self, queue, function, curl_hack):
-        """
-        function: function to execute, which takes the
-        callback (provided by this class) as an argument
-        """
-        threading.Thread.__init__(self)
-        self.function = function
-        self.queue = queue
-        self.die = False
-        self.curl_hack = curl_hack
-
-    def callback(self, data):
-        try:
-            if self.die:
-                raise Exception() # trigger termination
-            self.queue.put((1, data))
-        except:
-            if self.curl_hack:
-                # We can't raise exceptions, because the pycurl
-                # extension module will unconditionally print the
-                # exception itself, and not pass it up to the caller.
-                # Instead, just return a value that tells curl to
-                # abort.  (-1 would be best, in case we were given 0
-                # bytes, but the extension doesn't support that).
-                self.queue.put((2, sys.exc_info()))
-                return 0
-            raise
-
-    def run(self):
-        try:
-            result = self.function(self.callback)
-        except:
-            self.queue.put((2, sys.exc_info()))
-        else:
-            self.queue.put((0, result))
-
-@contextlib.contextmanager
-def Iteratorizer(function, curl_hack = False):
-    """
-    Context manager that takes a function expecting a callback,
-    and provides an iterable that yields the values passed to that
-    callback instead.
-
-    function: function to execute, which takes a callback
-    (provided by this context manager) as an argument
-
-        with iteratorizer(func) as it:
-            for i in it:
-                print 'callback was passed:', i
-        print 'function returned:', it.retval
-    """
-    queue = Queue.Queue(maxsize = 1)
-    thread = IteratorizerThread(queue, function, curl_hack)
-    thread.daemon = True
-    thread.start()
-
-    class iteratorizer_gen(object):
-        def __init__(self, queue):
-            self.queue = queue
-            self.retval = None
-
-        def __iter__(self):
-            return self
-
-        def next(self):
-            (typ, data) = self.queue.get()
-            if typ == 0:
-                # function has returned
-                self.retval = data
-                raise StopIteration
-            elif typ == 1:
-                # data is available
-                return data
-            else:
-                # callback raised an exception
-                raise data[0], data[1], data[2]
-
-    try:
-        yield iteratorizer_gen(queue)
-    finally:
-        # Ask the thread to die, if it's still running.
-        thread.die = True
-        while thread.isAlive():
-            try:
-                queue.get(True, 0.01)
-            except:
-                pass

nilmdb/utils/lock.py

@@ -0,0 +1,33 @@
+# File locking
+
+import warnings
+
+try:
+    import fcntl
+    import errno
+
+    def exclusive_lock(f):
+        """Acquire an exclusive lock.  Returns True on successful
+        lock, or False on error."""
+        try:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError as e:
+            if e.errno in (errno.EACCES, errno.EAGAIN):
+                return False
+            else: # pragma: no cover
+                raise
+        return True
+
+    def exclusive_unlock(f):
+        """Release an exclusive lock."""
+        fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+
+except ImportError: # pragma: no cover
+    def exclusive_lock(f):
+        """Dummy lock function -- does not lock!"""
+        warnings.warn("Pretending to lock " + str(f))
+        return True
+
+    def exclusive_unlock(f):
+        """Release an exclusive lock."""
+        return

nilmdb/utils/lrucache.py

@@ -5,7 +5,6 @@
 
 import collections
 import decorator
-import warnings
 
 def lru_cache(size = 10, onremove = None, keys = slice(None)):
     """Least-recently-used cache decorator.

nilmdb/utils/misc.py

@@ -1,8 +0,0 @@
-import itertools
-
-def pairwise(iterable):
-    "s -> (s0,s1), (s1,s2), ..., (sn,None)"
-    a, b = itertools.tee(iterable)
-    next(b, None)
-    return itertools.izip_longest(a, b)
-

nilmdb/utils/mustclose.py

@@ -12,15 +12,12 @@ def must_close(errorfile = sys.stderr, wrap_verify = False):
     already been called."""
     def class_decorator(cls):
 
-        # Helper to replace a class method with a wrapper function,
+        def wrap_class_method(wrapper):
-        # while maintaining argument specs etc.
+            try:
-        def wrap_class_method(wrapper_func):
+                orig = getattr(cls, wrapper.__name__).im_func
-            method = wrapper_func.__name__
+            except Exception:
-            if method in cls.__dict__:
+                orig = lambda x: None
-                orig = getattr(cls, method).im_func
+            setattr(cls, wrapper.__name__, decorator.decorator(wrapper, orig))
-            else:
-                orig = lambda self: None
-            setattr(cls, method, decorator.decorator(wrapper_func, orig))
 
         @wrap_class_method
         def __init__(orig, self, *args, **kwargs):
@@ -38,7 +35,8 @@ def must_close(errorfile = sys.stderr, wrap_verify = False):
 
         @wrap_class_method
         def close(orig, self, *args, **kwargs):
-            del self._must_close
+            if "_must_close" in self.__dict__:
+                del self._must_close
             return orig(self, *args, **kwargs)
 
         # Optionally wrap all other functions

nilmdb/utils/serializer.py

@@ -1,6 +1,10 @@
 import Queue
 import threading
 import sys
+import decorator
+import inspect
+import types
+import functools
 
 # This file provides a class that will wrap an object and serialize
 # all calls to its methods.  All calls to that object will be queued
@@ -12,8 +16,9 @@ import sys
 class SerializerThread(threading.Thread):
     """Thread that retrieves call information from the queue, makes the
     call, and returns the results."""
-    def __init__(self, call_queue):
+    def __init__(self, classname, call_queue):
         threading.Thread.__init__(self)
+        self.name = "Serializer-" + classname + "-" + self.name
         self.call_queue = call_queue
 
     def run(self):
@@ -22,51 +27,83 @@ class SerializerThread(threading.Thread):
             # Terminate if result_queue is None
             if result_queue is None:
                 return
+            exception = None
+            result = None
             try:
                 result = func(*args, **kwargs) # wrapped
             except:
-                result_queue.put((sys.exc_info(), None))
+                exception = sys.exc_info()
+            # Ensure we delete these before returning a result, so
+            # we don't unncessarily hold onto a reference while
+            # we're waiting for the next call.
+            del func, args, kwargs
+            result_queue.put((exception, result))
+            del exception, result
+
+def serializer_proxy(obj_or_type):
+    """Wrap the given object or type in a SerializerObjectProxy.
+
+    Returns a SerializerObjectProxy object that proxies all method
+    calls to the object, as well as attribute retrievals.
+
+    The proxied requests, including instantiation, are performed in a
+    single thread and serialized between caller threads.
+    """
+    class SerializerCallProxy(object):
+        def __init__(self, call_queue, func, objectproxy):
+            self.call_queue = call_queue
+            self.func = func
+            # Need to hold a reference to object proxy so it doesn't
+            # go away (and kill the thread) until after get called.
+            self.objectproxy = objectproxy
+        def __call__(self, *args, **kwargs):
+            result_queue = Queue.Queue()
+            self.call_queue.put((result_queue, self.func, args, kwargs))
+            ( exc_info, result ) = result_queue.get()
+            if exc_info is None:
+                return result
             else:
-                result_queue.put((None, result))
+                raise exc_info[0], exc_info[1], exc_info[2]
 
-class WrapCall(object):
+    class SerializerObjectProxy(object):
-    """Wrap a callable using the given queues"""
+        def __init__(self, obj_or_type, *args, **kwargs):
+            self.__object = obj_or_type
+            try:
+                if type(obj_or_type) in (types.TypeType, types.ClassType):
+                    classname = obj_or_type.__name__
+                else:
+                    classname = obj_or_type.__class__.__name__
+            except AttributeError: # pragma: no cover
+                classname = "???"
+            self.__call_queue = Queue.Queue()
+            self.__thread = SerializerThread(classname, self.__call_queue)
+            self.__thread.daemon = True
+            self.__thread.start()
+            self._thread_safe = True
 
-    def __init__(self, call_queue, result_queue, func):
+        def __getattr__(self, key):
-        self.call_queue = call_queue
+            if key.startswith("_SerializerObjectProxy__"): # pragma: no cover
-        self.result_queue = result_queue
+                raise AttributeError
-        self.func = func
+            attr = getattr(self.__object, key)
+            if not callable(attr):
+                getter = SerializerCallProxy(self.__call_queue, getattr, self)
+                return getter(self.__object, key)
+            r = SerializerCallProxy(self.__call_queue, attr, self)
+            return r
 
-    def __call__(self, *args, **kwargs):
+        def __call__(self, *args, **kwargs):
-        self.call_queue.put((self.result_queue, self.func, args, kwargs))
+            """Call this to instantiate the type, if a type was passed
-        ( exc_info, result ) = self.result_queue.get()
+            to serializer_proxy.  Otherwise, pass the call through."""
-        if exc_info is None:
+            ret = SerializerCallProxy(self.__call_queue,
-            return result
+                                      self.__object, self)(*args, **kwargs)
-        else:
+            if type(self.__object) in (types.TypeType, types.ClassType):
-            raise exc_info[0], exc_info[1], exc_info[2]
+                # Instantiation
+                self.__object = ret
+                return self
+            return ret
 
-class WrapObject(object):
+        def __del__(self):
-    """Wrap all calls to methods in a target object with WrapCall"""
+            self.__call_queue.put((None, None, None, None))
+            self.__thread.join()
 
-    def __init__(self, target):
+    return SerializerObjectProxy(obj_or_type)
-        self.__wrap_target = target
-        self.__wrap_call_queue = Queue.Queue()
-        self.__wrap_serializer = SerializerThread(self.__wrap_call_queue)
-        self.__wrap_serializer.daemon = True
-        self.__wrap_serializer.start()
-
-    def __getattr__(self, key):
-        """Wrap methods of self.__wrap_target in a WrapCall instance"""
-        func = getattr(self.__wrap_target, key)
-        if not callable(func):
-            raise TypeError("Can't serialize attribute %r (type: %s)"
-                            % (key, type(func)))
-        result_queue = Queue.Queue()
-        return WrapCall(self.__wrap_call_queue, result_queue, func)
-
-    def __del__(self):
-        self.__wrap_call_queue.put((None, None, None, None))
-        self.__wrap_serializer.join()
-
-# Just an alias
-Serializer = WrapObject

nilmdb/utils/threadsafety.py

@@ -0,0 +1,109 @@
+from nilmdb.utils.printf import *
+import threading
+import warnings
+import types
+
+def verify_proxy(obj_or_type, exception = False, check_thread = True,
+                 check_concurrent = True):
+    """Wrap the given object or type in a VerifyObjectProxy.
+
+    Returns a VerifyObjectProxy that proxies all method calls to the
+    given object, as well as attribute retrievals.
+
+    When calling methods, the following checks are performed.  If
+    exception is True, an exception is raised.  Otherwise, a warning
+    is printed.
+
+    check_thread = True     # Warn/fail if two different threads call methods.
+    check_concurrent = True # Warn/fail if two functions are concurrently
+                            # run through this proxy
+    """
+    class Namespace(object):
+        pass
+    class VerifyCallProxy(object):
+        def __init__(self, func, parent_namespace):
+            self.func = func
+            self.parent_namespace = parent_namespace
+
+        def __call__(self, *args, **kwargs):
+            p = self.parent_namespace
+            this = threading.current_thread()
+            try:
+                callee = self.func.__name__
+            except AttributeError:
+                callee = "???"
+
+            if p.thread is None:
+                p.thread = this
+                p.thread_callee = callee
+
+            if check_thread and p.thread != this:
+                err = sprintf("unsafe threading: %s called %s.%s,"
+                              " but %s called %s.%s",
+                              p.thread.name, p.classname, p.thread_callee,
+                              this.name, p.classname, callee)
+                if exception:
+                    raise AssertionError(err)
+                else: # pragma: no cover
+                    warnings.warn(err)
+
+            need_concur_unlock = False
+            if check_concurrent:
+                if p.concur_lock.acquire(False) == False:
+                    err = sprintf("unsafe concurrency: %s called %s.%s "
+                                  "while %s is still in %s.%s",
+                                  this.name, p.classname, callee,
+                                  p.concur_tname, p.classname, p.concur_callee)
+                    if exception:
+                        raise AssertionError(err)
+                    else: # pragma: no cover
+                        warnings.warn(err)
+                else:
+                    p.concur_tname = this.name
+                    p.concur_callee = callee
+                    need_concur_unlock = True
+
+            try:
+                ret = self.func(*args, **kwargs)
+            finally:
+                if need_concur_unlock:
+                    p.concur_lock.release()
+            return ret
+
+    class VerifyObjectProxy(object):
+        def __init__(self, obj_or_type, *args, **kwargs):
+            p = Namespace()
+            self.__ns = p
+            p.thread = None
+            p.thread_callee = None
+            p.concur_lock = threading.Lock()
+            p.concur_tname = None
+            p.concur_callee = None
+            self.__obj = obj_or_type
+            try:
+                if type(obj_or_type) in (types.TypeType, types.ClassType):
+                    p.classname = self.__obj.__name__
+                else:
+                    p.classname = self.__obj.__class__.__name__
+            except AttributeError: # pragma: no cover
+                p.classname = "???"
+
+        def __getattr__(self, key):
+            if key.startswith("_VerifyObjectProxy__"): # pragma: no cover
+                raise AttributeError
+            attr = getattr(self.__obj, key)
+            if not callable(attr):
+                return VerifyCallProxy(getattr, self.__ns)(self.__obj, key)
+            return VerifyCallProxy(attr, self.__ns)
+
+        def __call__(self, *args, **kwargs):
+            """Call this to instantiate the type, if a type was passed
+            to verify_proxy.  Otherwise, pass the call through."""
+            ret = VerifyCallProxy(self.__obj, self.__ns)(*args, **kwargs)
+            if type(self.__obj) in (types.TypeType, types.ClassType):
+                # Instantiation
+                self.__obj = ret
+                return self
+            return ret
+
+    return VerifyObjectProxy(obj_or_type)

nilmdb/utils/time.py

@@ -0,0 +1,134 @@
+from __future__ import absolute_import
+
+from nilmdb.utils import datetime_tz
+import re
+import time
+
+# Range
+min_timestamp = (-2**63)
+max_timestamp = (2**63 - 1)
+
+# Smallest representable step
+epsilon = 1
+
+def string_to_timestamp(str):
+    """Convert a string that represents an integer number of microseconds
+    since epoch."""
+    try:
+        # Parse a string like "1234567890123456" and return an integer
+        return int(str)
+    except ValueError:
+        # Try parsing as a float, in case it's "1234567890123456.0"
+        return int(round(float(str)))
+
+def timestamp_to_string(timestamp):
+    """Convert a timestamp (integer microseconds since epoch) to a string"""
+    if isinstance(timestamp, float):
+        return str(int(round(timestamp)))
+    else:
+        return str(timestamp)
+
+def timestamp_to_human(timestamp):
+    """Convert a timestamp (integer microseconds since epoch) to a
+    human-readable string, using the local timezone for display
+    (e.g. from the TZ env var)."""
+    if timestamp == min_timestamp:
+        return "(minimum)"
+    if timestamp == max_timestamp:
+        return "(maximum)"
+    dt = datetime_tz.datetime_tz.fromtimestamp(timestamp_to_unix(timestamp))
+    return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
+
+def unix_to_timestamp(unix):
+    """Convert a Unix timestamp (floating point seconds since epoch)
+    into a NILM timestamp (integer microseconds since epoch)"""
+    return int(round(unix * 1e6))
+seconds_to_timestamp = unix_to_timestamp
+
+def timestamp_to_unix(timestamp):
+    """Convert a NILM timestamp (integer microseconds since epoch)
+    into a Unix timestamp (floating point seconds since epoch)"""
+    return timestamp / 1e6
+timestamp_to_seconds = timestamp_to_unix
+
+def rate_to_period(hz, cycles = 1):
+    """Convert a rate (in Hz) to a period (in timestamp units).
+    Returns an integer."""
+    period = unix_to_timestamp(cycles) / float(hz)
+    return int(round(period))
+
+def parse_time(toparse):
+    """
+    Parse a free-form time string and return a nilmdb timestamp
+    (integer seconds since epoch).  If the string doesn't contain a
+    timestamp, the current local timezone is assumed (e.g. from the TZ
+    env var).
+    """
+    if toparse == "min":
+        return min_timestamp
+    if toparse == "max":
+        return max_timestamp
+
+    # If it starts with @, treat it as a NILM timestamp
+    # (integer microseconds since epoch)
+    try:
+        if toparse[0] == '@':
+            return int(toparse[1:])
+    except (ValueError, KeyError, IndexError):
+        pass
+
+    # If string isn't "now" and doesn't contain at least 4 digits,
+    # consider it invalid.  smartparse might otherwise accept
+    # empty strings and strings with just separators.
+    if toparse != "now" and len(re.findall(r"\d", toparse)) < 4:
+        raise ValueError("not enough digits for a timestamp")
+
+    # Try to just parse the time as given
+    try:
+        return unix_to_timestamp(datetime_tz.datetime_tz.
+                                 smartparse(toparse).totimestamp())
+    except (ValueError, OverflowError):
+        pass
+
+    # If it's parseable as a float, treat it as a Unix or NILM
+    # timestamp based on its range.
+    try:
+        val = float(toparse)
+        # range is from about year 2001 - 2128
+        if val > 1e9 and val < 5e9:
+            return unix_to_timestamp(val)
+        if val > 1e15 and val < 5e15:
+            return val
+    except ValueError:
+        pass
+
+    # Try to extract a substring in a condensed format that we expect
+    # to see in a filename or header comment
+    res = re.search(r"(^|[^\d])("            # non-numeric or SOL
+                    r"(199\d|2\d\d\d)"       # year
+                    r"[-/]?"                 # separator
+                    r"(0[1-9]|1[012])"       # month
+                    r"[-/]?"                 # separator
+                    r"([012]\d|3[01])"       # day
+                    r"[-T ]?"                # separator
+                    r"([01]\d|2[0-3])"       # hour
+                    r"[:]?"                  # separator
+                    r"([0-5]\d)"             # minute
+                    r"[:]?"                  # separator
+                    r"([0-5]\d)?"            # second
+                    r"([-+]\d\d\d\d)?"       # timezone
+                    r")", toparse)
+    if res is not None:
+        try:
+            return unix_to_timestamp(datetime_tz.datetime_tz.
+                                     smartparse(res.group(2)).totimestamp())
+        except ValueError:
+            pass
+
+    # Could also try to successively parse substrings, but let's
+    # just give up for now.
+    raise ValueError("unable to parse timestamp")
+
+def now():
+    """Return current timestamp"""
+    return unix_to_timestamp(time.time())

nilmdb/utils/timer.py

@@ -2,10 +2,11 @@
 
 # Simple timer to time a block of code, for optimization debugging
 # use like:
-#   with nilmdb.Timer("flush"):
+#   with nilmdb.utils.Timer("flush"):
 #       foo.flush()
 
 from __future__ import print_function
+from __future__ import absolute_import
 import contextlib
 import time
 

nilmdb/utils/timestamper.py

@@ -1,21 +1,18 @@
 """File-like objects that add timestamps to the input lines"""
 
 from nilmdb.utils.printf import *
-from nilmdb.utils import datetime_tz
+import nilmdb.utils.time
-
-import time
-import os
 
 class Timestamper(object):
     """A file-like object that adds timestamps to lines of an input file."""
-    def __init__(self, file, ts_iter):
+    def __init__(self, infile, ts_iter):
         """file: filename, or another file-like object
            ts_iter: iterator that returns a timestamp string for
            each line of the file"""
-        if isinstance(file, basestring):
+        if isinstance(infile, basestring):
-            self.file = open(file, "r")
+            self.file = open(infile, "r")
         else:
-            self.file = file
+            self.file = infile
         self.ts_iter = ts_iter
 
     def close(self):
@@ -54,7 +51,7 @@ class Timestamper(object):
 
 class TimestamperRate(Timestamper):
     """Timestamper that uses a start time and a fixed rate"""
-    def __init__(self, file, start, rate, end = None):
+    def __init__(self, infile, start, rate, end = None):
         """
         file: file name or object
 
@@ -64,44 +61,33 @@ class TimestamperRate(Timestamper):
 
         end: If specified, raise StopIteration before outputting a value
              greater than this."""
+        timestamp_to_string = nilmdb.utils.time.timestamp_to_string
+        rate_to_period = nilmdb.utils.time.rate_to_period
         def iterator(start, rate, end):
             n = 0
             rate = float(rate)
             while True:
-                now = start + n / rate
+                now = start + rate_to_period(rate, n)
                 if end and now >= end:
                     raise StopIteration
-                yield sprintf("%.6f ", start + n / rate)
+                yield timestamp_to_string(now) + " "
                 n += 1
-        # Handle case where we're passed a datetime or datetime_tz object
+        Timestamper.__init__(self, infile, iterator(start, rate, end))
-        if "totimestamp" in dir(start):
-            start = start.totimestamp()
-        Timestamper.__init__(self, file, iterator(start, rate, end))
         self.start = start
         self.rate = rate
     def __str__(self):
-        start = datetime_tz.datetime_tz.fromtimestamp(self.start)
-        start = start.strftime("%a, %d %b %Y %H:%M:%S %Z")
         return sprintf("TimestamperRate(..., start=\"%s\", rate=%g)",
-                       str(start), self.rate)
+                       nilmdb.utils.time.timestamp_to_human(self.start),
+                       self.rate)
 
 class TimestamperNow(Timestamper):
     """Timestamper that uses current time"""
-    def __init__(self, file):
+    def __init__(self, infile):
+        timestamp_to_string = nilmdb.utils.time.timestamp_to_string
+        get_now = nilmdb.utils.time.now
         def iterator():
             while True:
-                now = datetime_tz.datetime_tz.utcnow().totimestamp()
+                yield timestamp_to_string(get_now()) + " "
-                yield sprintf("%.6f ", now)
+        Timestamper.__init__(self, infile, iterator())
-        Timestamper.__init__(self, file, iterator())
     def __str__(self):
         return "TimestamperNow(...)"
-
-class TimestamperNull(Timestamper):
-    """Timestamper that adds nothing to each line"""
-    def __init__(self, file):
-        def iterator():
-            while True:
-                yield ""
-        Timestamper.__init__(self, file, iterator())
-    def __str__(self):
-        return "TimestamperNull(...)"

nilmdb/utils/urllib.py

@@ -1,37 +0,0 @@
-from __future__ import absolute_import
-from urllib import quote_plus, _is_unicode
-
-# urllib.urlencode insists on encoding Unicode as ASCII.  This is based
-# on that function, except we always encode it as UTF-8 instead.
-
-def urlencode(query):
-    """Encode a dictionary into a URL query string.
-
-    If any values in the query arg are sequences, each sequence
-    element is converted to a separate parameter.
-    """
-
-    query = query.items()
-
-    l = []
-    for k, v in query:
-        k = quote_plus(str(k))
-        if isinstance(v, str):
-            v = quote_plus(v)
-            l.append(k + '=' + v)
-        elif _is_unicode(v):
-            v = quote_plus(v.encode("utf-8","strict"))
-            l.append(k + '=' + v)
-        else:
-            try:
-                # is this a sufficient test for sequence-ness?
-                len(v)
-            except TypeError:
-                # not a sequence
-                v = quote_plus(str(v))
-                l.append(k + '=' + v)
-            else:
-                # loop over the sequence
-                for elt in v:
-                    l.append(k + '=' + quote_plus(str(elt)))
-    return '&'.join(l)

nilmtool.py

@@ -1,6 +0,0 @@
-#!/usr/bin/python
-
-import nilmdb
-import sys
-
-nilmdb.cmdline.Cmdline(sys.argv[1:]).run()

runserver.py

@@ -1,35 +0,0 @@
-#!/usr/bin/python
-
-import nilmdb
-import argparse
-
-formatter = argparse.ArgumentDefaultsHelpFormatter
-parser = argparse.ArgumentParser(description='Run the NILM server',
-                                 formatter_class = formatter)
-parser.add_argument('-p', '--port', help='Port number', type=int, default=12380)
-parser.add_argument('-d', '--database', help='Database directory', default="db")
-parser.add_argument('-y', '--yappi', help='Run with yappi profiler',
-                    action='store_true')
-args = parser.parse_args()
-
-# Start web app on a custom port
-db = nilmdb.NilmDB(args.database)
-server = nilmdb.Server(db, host = "127.0.0.1",
-                       port = args.port,
-                       embedded = False)
-
-
-if args.yappi:
-    print "Running in yappi"
-    try:
-        import yappi
-        yappi.start()
-        server.start(blocking = True)
-    finally:
-        yappi.stop()
-        print "Try: yappi.print_stats(sort_type=yappi.SORTTYPE_TTOT,limit=50)"
-        from IPython import embed
-        embed()
-else:
-    server.start(blocking = True)
-db.close()

setup.cfg

@@ -20,6 +20,7 @@ cover-erase=1
 stop=1
 verbosity=2
 tests=tests
+#tests=tests/test_threadsafety.py
 #tests=tests/test_bulkdata.py
 #tests=tests/test_mustclose.py
 #tests=tests/test_lrucache.py

setup.py

@@ -1,5 +1,11 @@
 #!/usr/bin/python
 
+# To release a new version, tag it:
+#   git tag -a nilmdb-1.1 -m "Version 1.1"
+#   git push --tags
+# Then just package it up:
+#   python setup.py sdist
+
 # This is supposed to be using Distribute:
 #
 #   distutils provides a "setup" method.
@@ -9,32 +15,109 @@
 # So we don't really know if this is using the old setuptools or the
 # Distribute-provided version of setuptools.
 
-from setuptools import setup, find_packages
+import traceback
-from distutils.extension import Extension
+import sys
+import os
 
-from Cython.Build import cythonize
+try:
+    from setuptools import setup, find_packages
+    from distutils.extension import Extension
+    import distutils.version
+except ImportError:
+    traceback.print_exc()
+    print "Please install the prerequisites listed in README.txt"
+    sys.exit(1)
+
+# Versioneer manages version numbers from git tags.
+# https://github.com/warner/python-versioneer
+import versioneer
+versioneer.versionfile_source = 'nilmdb/_version.py'
+versioneer.versionfile_build = 'nilmdb/_version.py'
+versioneer.tag_prefix = 'nilmdb-'
+versioneer.parentdir_prefix = 'nilmdb-'
 
 # Hack to workaround logging/multiprocessing issue:
 # https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
 try: import multiprocessing
-except: pass
+except Exception: pass
 
-# Build cython modules.
+# Use Cython if it's new enough, otherwise use preexisting C files.
-cython_modules = cythonize("**/*.pyx")
+cython_modules = [ 'nilmdb.server.interval',
+                   'nilmdb.server.rbtree' ]
+try:
+    import Cython
+    from Cython.Build import cythonize
+    if (distutils.version.LooseVersion(Cython.__version__) <
+        distutils.version.LooseVersion("0.16")):
+        print "Cython version", Cython.__version__, "is too old; not using it."
+        raise ImportError()
+    use_cython = True
+except ImportError:
+    use_cython = False
+
+ext_modules = [ Extension('nilmdb.server.rocket', ['nilmdb/server/rocket.c' ]) ]
+for modulename in cython_modules:
+    filename = modulename.replace('.','/')
+    if use_cython:
+        ext_modules.extend(cythonize(filename + ".pyx"))
+    else:
+        cfile = filename + ".c"
+        if not os.path.exists(cfile):
+            raise Exception("Missing source file " + cfile + ".  "
+                            "Try installing cython >= 0.16.")
+        ext_modules.append(Extension(modulename, [ cfile ]))
+
+# We need a MANIFEST.in.  Generate it here rather than polluting the
+# repository with yet another setup-related file.
+with open("MANIFEST.in", "w") as m:
+    m.write("""
+# Root
+include README.txt
+include setup.cfg
+include setup.py
+include versioneer.py
+include Makefile
+include .coveragerc
+include .pylintrc
+
+# Cython files -- include source.
+recursive-include nilmdb/server *.pyx *.pyxdep *.pxd
+
+# Tests
+recursive-include tests *.py
+recursive-include tests/data *
+include tests/test.order
+
+# Docs
+recursive-include docs Makefile *.md
+
+# Extras
+recursive-include extras *
+""")
 
 # Run setup
 setup(name='nilmdb',
-      version = '1.0',
+      version = versioneer.get_version(),
+      cmdclass = versioneer.get_cmdclass(),
       url = 'https://git.jim.sh/jim/lees/nilmdb.git',
       author = 'Jim Paris',
+      description = "NILM Database",
+      long_description = "NILM Database",
+      license = "Proprietary",
       author_email = 'jim@jtan.com',
       tests_require = [ 'nose',
                         'coverage',
+                        'numpy',
                         ],
-      setup_requires = [ 'cython',
+      setup_requires = [ 'distribute',
-                        ],
+                         ],
-      install_requires = [ 'distribute',
+      install_requires = [ 'decorator',
-                           'decorator',
+                           'cherrypy >= 3.2',
+                           'simplejson',
+                           'python-dateutil',
+                           'pytz',
+                           'psutil >= 0.3.0',
+                           'requests >= 1.1.0, < 2.0.0',
                            ],
       packages = [ 'nilmdb',
                    'nilmdb.utils',
@@ -42,7 +125,14 @@ setup(name='nilmdb',
                    'nilmdb.server',
                    'nilmdb.client',
                    'nilmdb.cmdline',
+                   'nilmdb.scripts',
                    ],
-      ext_modules = cython_modules,
+      entry_points = {
+          'console_scripts': [
+              'nilmtool = nilmdb.scripts.nilmtool:main',
+              'nilmdb-server = nilmdb.scripts.nilmdb_server:main',
+              ],
+          },
+      ext_modules = ext_modules,
       zip_safe = False,
       )

tests/data/extract-1

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
 # start: Fri, 23 Mar 2012 10:00:30.000000 +0000
 # end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+1332496830000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-2

@@ -1,119 +1,119 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-3

@@ -1 +1 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03

tests/data/extract-4

@@ -1,2 +1,2 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03

tests/data/extract-5

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
 # start: Fri, 23 Mar 2012 10:00:30.000000 +0000
 # end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-6

@@ -1,120 +1,120 @@
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-7

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
-# start: 1332496830.0
+# start: 1332496830000000
-# end: 1332496830.999
+# end: 1332496830999000
-1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+1332496830000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-8

@@ -0,0 +1,28 @@
+# interval-start 1332496919900000
+1332496919900000 2.523050e+05 2.254020e+05 4.779410e+03 3.638030e+03 8.138070e+03 4.334460e+03 1.083780e+03 3.743730e+03
+1332496919908333 2.551190e+05 2.237870e+05 5.965640e+03 2.076350e+03 9.468790e+03 3.693880e+03 1.247860e+03 3.393680e+03
+1332496919916667 2.616370e+05 2.247980e+05 4.848970e+03 2.315620e+03 9.323300e+03 4.225460e+03 1.805780e+03 2.593050e+03
+1332496919925000 2.606460e+05 2.251300e+05 3.061360e+03 3.951840e+03 7.662910e+03 5.341410e+03 1.986520e+03 2.276780e+03
+1332496919933333 2.559710e+05 2.235030e+05 4.096030e+03 3.296970e+03 7.827080e+03 5.452120e+03 2.492520e+03 2.929450e+03
+1332496919941667 2.579260e+05 2.217080e+05 5.472320e+03 1.555700e+03 8.495760e+03 4.491140e+03 2.379780e+03 3.741710e+03
+1332496919950000 2.610180e+05 2.242350e+05 4.669770e+03 1.876190e+03 8.366680e+03 3.677510e+03 9.021690e+02 3.549040e+03
+1332496919958333 2.569150e+05 2.274650e+05 2.785070e+03 3.751930e+03 7.440320e+03 3.964860e+03 -3.227860e+02 2.460890e+03
+1332496919966667 2.509510e+05 2.262000e+05 3.772710e+03 3.131950e+03 8.159860e+03 4.539860e+03 7.375190e+02 2.126750e+03
+1332496919975000 2.556710e+05 2.223720e+05 5.826200e+03 8.715560e+02 9.120240e+03 4.545110e+03 2.804310e+03 2.721000e+03
+1332496919983333 2.649730e+05 2.214860e+05 5.839130e+03 4.659180e+02 8.628300e+03 3.934870e+03 2.972490e+03 3.773730e+03
+1332496919991667 2.652170e+05 2.233920e+05 3.718770e+03 2.834970e+03 7.209900e+03 3.460260e+03 1.324930e+03 4.075960e+03
+# interval-end 1332496919991668
+# interval-start 1332496920000000
+1332496920000000 2.564370e+05 2.244300e+05 4.011610e+03 3.475340e+03 7.495890e+03 3.388940e+03 2.613970e+02 3.731260e+03
+1332496920008333 2.539630e+05 2.241670e+05 5.621070e+03 1.548010e+03 9.165170e+03 3.522930e+03 1.058930e+03 2.996960e+03
+1332496920016667 2.585080e+05 2.249300e+05 6.011400e+03 8.188660e+02 9.039950e+03 4.482440e+03 2.490390e+03 2.679340e+03
+1332496920025000 2.596270e+05 2.260220e+05 4.474500e+03 2.423020e+03 7.414190e+03 5.071970e+03 2.439380e+03 2.962960e+03
+1332496920033333 2.551870e+05 2.246320e+05 4.738570e+03 3.398040e+03 7.395120e+03 4.726450e+03 1.839030e+03 3.393530e+03
+1332496920041667 2.571020e+05 2.216230e+05 6.144130e+03 1.441090e+03 8.756480e+03 3.495320e+03 1.869940e+03 3.752530e+03
+1332496920050000 2.636530e+05 2.217700e+05 6.221770e+03 7.389620e+02 9.547600e+03 2.666820e+03 1.462660e+03 3.332570e+03
+1332496920058333 2.636130e+05 2.252560e+05 4.477120e+03 2.437450e+03 8.510210e+03 3.855630e+03 9.594420e+02 2.387180e+03
+1332496920066667 2.553500e+05 2.262640e+05 4.283720e+03 3.923940e+03 7.912470e+03 5.466520e+03 1.284990e+03 2.093720e+03
+1332496920075000 2.527270e+05 2.246090e+05 5.851930e+03 2.491980e+03 8.540630e+03 5.623050e+03 2.339780e+03 3.007140e+03
+1332496920083333 2.584750e+05 2.235780e+05 5.924870e+03 1.394480e+03 8.779620e+03 4.544180e+03 2.132030e+03 3.849760e+03
+1332496920091667 2.615630e+05 2.246090e+05 4.336140e+03 2.455750e+03 8.055380e+03 3.469110e+03 6.278730e+02 3.664200e+03
+# interval-end 1332496920100000

tests/data/prep-20120323T1000

@@ -1,3 +1,4 @@
+# comments are cool?
 2.66568e+05  2.24029e+05  5.16140e+03  2.52517e+03  8.35084e+03  3.72470e+03  1.35534e+03  2.03900e+03  
 2.57914e+05  2.27183e+05  4.30368e+03  4.13080e+03  7.25535e+03  4.89047e+03  1.63859e+03  1.93496e+03  
 2.51717e+05  2.26047e+05  5.99445e+03  3.49363e+03  8.07250e+03  5.08267e+03  2.26917e+03  2.86231e+03  

tests/data/prep-20120323T1004-badtimes

@@ -1,11 +1,11 @@
-1332497040.000000 2.56439e+05  2.24775e+05  2.92897e+03  4.66646e+03  7.58491e+03  3.57351e+03  -4.34171e+02  2.98819e+03  
+1332497040000000 2.56439e+05  2.24775e+05  2.92897e+03  4.66646e+03  7.58491e+03  3.57351e+03  -4.34171e+02  2.98819e+03  
-1332497040.010000 2.51903e+05  2.23202e+05  4.23696e+03  3.49363e+03  8.53493e+03  4.29416e+03  8.49573e+02  2.38189e+03  
+1332497040010000 2.51903e+05  2.23202e+05  4.23696e+03  3.49363e+03  8.53493e+03  4.29416e+03  8.49573e+02  2.38189e+03  
-1332497040.020000 2.57625e+05  2.20247e+05  5.47017e+03  1.35872e+03  9.18903e+03  4.56136e+03  2.65599e+03  2.60912e+03  
+1332497040020000 2.57625e+05  2.20247e+05  5.47017e+03  1.35872e+03  9.18903e+03  4.56136e+03  2.65599e+03  2.60912e+03  
-1332497040.030000 2.63375e+05  2.20706e+05  4.51842e+03  1.80758e+03  8.17208e+03  4.17463e+03  2.57884e+03  3.32848e+03  
+1332497040030000 2.63375e+05  2.20706e+05  4.51842e+03  1.80758e+03  8.17208e+03  4.17463e+03  2.57884e+03  3.32848e+03  
-1332497040.040000 2.59221e+05  2.22346e+05  2.98879e+03  3.66264e+03  6.87274e+03  3.94223e+03  1.25928e+03  3.51786e+03  
+1332497040040000 2.59221e+05  2.22346e+05  2.98879e+03  3.66264e+03  6.87274e+03  3.94223e+03  1.25928e+03  3.51786e+03  
-1332497040.050000 2.51918e+05  2.22281e+05  4.22677e+03  2.84764e+03  7.78323e+03  3.81659e+03  8.04944e+02  3.46314e+03  
+1332497040050000 2.51918e+05  2.22281e+05  4.22677e+03  2.84764e+03  7.78323e+03  3.81659e+03  8.04944e+02  3.46314e+03  
-1332497040.050000 2.54478e+05  2.21701e+05  5.61366e+03  1.02262e+03  9.26581e+03  3.50152e+03  1.29331e+03  3.07271e+03  
+1332497040050000 2.54478e+05  2.21701e+05  5.61366e+03  1.02262e+03  9.26581e+03  3.50152e+03  1.29331e+03  3.07271e+03  
-1332497040.060000 2.59568e+05  2.22945e+05  4.97190e+03  1.28250e+03  8.62081e+03  4.06316e+03  1.85717e+03  2.61990e+03  
+1332497040060000 2.59568e+05  2.22945e+05  4.97190e+03  1.28250e+03  8.62081e+03  4.06316e+03  1.85717e+03  2.61990e+03  
-1332497040.070000 2.57269e+05  2.23697e+05  3.60527e+03  3.05749e+03  7.22363e+03  4.90330e+03  1.93736e+03  2.35357e+03  
+1332497040070000 2.57269e+05  2.23697e+05  3.60527e+03  3.05749e+03  7.22363e+03  4.90330e+03  1.93736e+03  2.35357e+03  
-1332497040.080000 2.52274e+05  2.21438e+05  5.01228e+03  2.86309e+03  7.87115e+03  4.80448e+03  2.18291e+03  2.93397e+03  
+1332497040080000 2.52274e+05  2.21438e+05  5.01228e+03  2.86309e+03  7.87115e+03  4.80448e+03  2.18291e+03  2.93397e+03  
-1332497040.090000 2.56468e+05  2.19205e+05  6.29804e+03  8.09467e+02  9.12895e+03  3.52055e+03  2.16980e+03  3.88739e+03  
+1332497040090000 2.56468e+05  2.19205e+05  6.29804e+03  8.09467e+02  9.12895e+03  3.52055e+03  2.16980e+03  3.88739e+03  

tests/runtests.py

@@ -6,6 +6,9 @@ import sys
 import glob
 from collections import OrderedDict
 
+# Change into parent dir
+os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/..")
+
 class JimOrderPlugin(nose.plugins.Plugin):
     """When searching for tests and encountering a directory that
     contains a 'test.order' file, run tests listed in that file, in the
@@ -21,7 +24,7 @@ class JimOrderPlugin(nose.plugins.Plugin):
                     name, workingDir=loader.workingDir)
                 try:
                     order = os.path.join(addr.filename, "test.order")
-                except:
+                except Exception:
                     order = None
                 if order and os.path.exists(order):
                     files = []

tests/test.order

@@ -1,18 +1,18 @@
 test_printf.py
+test_threadsafety.py
 test_lrucache.py
 test_mustclose.py
 
 test_serializer.py
-test_iteratorizer.py
 
 test_timestamper.py
-test_layout.py
 test_rbtree.py
 test_interval.py
 
 test_bulkdata.py
 test_nilmdb.py
 test_client.py
+test_numpyclient.py
 test_cmdline.py
 
 test_*.py

tests/test_bulkdata.py

@@ -30,13 +30,16 @@ class TestBulkData(object):
         else:
             data = BulkData(db, file_size = size, files_per_dir = files)
 
+        # Try opening it again (should result in locking error)
+        with assert_raises(IOError) as e:
+            data2 = BulkData(db)
+        in_("already locked by another process", str(e.exception))
+
         # create empty
         with assert_raises(ValueError):
             data.create("/foo", "uint16_8")
         with assert_raises(ValueError):
             data.create("foo/bar", "uint16_8")
-        with assert_raises(ValueError):
-            data.create("/foo/bar", "uint8_8")
         data.create("/foo/bar", "uint16_8")
         data.create(u"/foo/baz/quux", "float64_16")
         with assert_raises(ValueError):
@@ -51,24 +54,40 @@ class TestBulkData(object):
             nodes.append(data.getnode("/foo/baz/quux"))
         del nodes
 
+        def get_node_slice(key):
+            if isinstance(key, slice):
+                return [ node.get_data(x, x+1) for x in
+                         xrange(*key.indices(node.nrows)) ]
+            return node.get_data(key, key+1)
+
         # Test node
         node = data.getnode("/foo/bar")
         with assert_raises(IndexError):
-            x = node[0]
+            x = get_node_slice(0)
+        with assert_raises(IndexError):
+            x = node[0] # timestamp
         raw = []
         for i in range(1000):
-            raw.append([10000+i, 1, 2, 3, 4, 5, 6, 7, 8 ])
+            raw.append("%d 1 2 3 4 5 6 7 8\n" % (10000 + i))
-        node.append(raw[0:1])
+        node.append_data("".join(raw[0:1]), 0, 50000)
-        node.append(raw[1:100])
+        node.append_data("".join(raw[1:100]), 0, 50000)
-        node.append(raw[100:])
+        node.append_data("".join(raw[100:]), 0, 50000)
 
         misc_slices = [ 0, 100, slice(None), slice(0), slice(10),
                         slice(5,10), slice(3,None), slice(3,-3),
                         slice(20,10), slice(200,100,-1), slice(None,0,-1),
                         slice(100,500,5) ]
+
         # Extract slices
         for s in misc_slices:
-            eq_(node[s], raw[s])
+            eq_(get_node_slice(s), raw[s])
+
+        # Extract misc slices while appending, to make sure the
+        # data isn't being added in the middle of the file
+        for s in [2, slice(1,5), 2, slice(1,5)]:
+            node.append_data("0 0 0 0 0 0 0 0 0\n", 0, 50000)
+            raw.append("0 0 0 0 0 0 0 0 0\n")
+            eq_(get_node_slice(s), raw[s])
 
         # Get some coverage of remove; remove is more fully tested
         # in cmdline
@@ -86,7 +105,7 @@ class TestBulkData(object):
 
         # Extract slices
         for s in misc_slices:
-            eq_(node[s], raw[s])
+            eq_(get_node_slice(s), raw[s])
 
         # destroy
         with assert_raises(ValueError):

tests/test_client.py

@@ -1,11 +1,14 @@
 # -*- coding: utf-8 -*-
 
-import nilmdb
+import nilmdb.server
+import nilmdb.client
+
 from nilmdb.utils.printf import *
 from nilmdb.utils import timestamper
 from nilmdb.client import ClientError, ServerError
 from nilmdb.utils import datetime_tz
 
+from nose.plugins.skip import SkipTest
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
@@ -18,10 +21,14 @@ import simplejson as json
 import unittest
 import warnings
 import resource
+import time
+import re
+import struct
 
 from testutil.helpers import *
 
 testdb = "tests/client-testdb"
+testurl = "http://localhost:32180/"
 
 def setup_module():
     global test_server, test_db
@@ -29,11 +36,11 @@ def setup_module():
     recursive_unlink(testdb)
 
     # Start web app on a custom port
-    test_db = nilmdb.NilmDB(testdb, sync = False)
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(testdb)
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
+    test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+                                       port = 32180, stoppable = False,
-                                fast_shutdown = True,
+                                       fast_shutdown = True,
-                                force_traceback = False)
+                                       force_traceback = True)
     test_server.start(blocking = False)
 
 def teardown_module():
@@ -44,65 +51,75 @@ def teardown_module():
 
 class TestClient(object):
 
-    def test_client_1_basic(self):
+    def test_client_01_basic(self):
         # Test a fake host
-        client = nilmdb.Client(url = "http://localhost:1/")
+        client = nilmdb.client.Client(url = "http://localhost:1/")
-        with assert_raises(nilmdb.client.ServerError):
-            client.version()
-
-        # Trigger same error with a PUT request
-        client = nilmdb.Client(url = "http://localhost:1/")
         with assert_raises(nilmdb.client.ServerError):
             client.version()
+        client.close()
 
         # Then a fake URL on a real host
-        client = nilmdb.Client(url = "http://localhost:12380/fake/")
+        client = nilmdb.client.Client(url = "http://localhost:32180/fake/")
         with assert_raises(nilmdb.client.ClientError):
             client.version()
+        client.close()
 
         # Now a real URL with no http:// prefix
-        client = nilmdb.Client(url = "localhost:12380")
+        client = nilmdb.client.Client(url = "localhost:32180")
         version = client.version()
+        client.close()
 
         # Now use the real URL
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
         version = client.version()
-        eq_(distutils.version.StrictVersion(version),
+        eq_(distutils.version.LooseVersion(version),
-            distutils.version.StrictVersion(test_server.version))
+            distutils.version.LooseVersion(test_server.version))
 
         # Bad URLs should give 404, not 500
         with assert_raises(ClientError):
             client.http.get("/stream/create")
+        client.close()
 
-    def test_client_2_createlist(self):
+    def test_client_02_createlist(self):
         # Basic stream tests, like those in test_nilmdb:test_stream
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
 
         # Database starts empty
         eq_(client.stream_list(), [])
 
         # Bad path
         with assert_raises(ClientError):
-            client.stream_create("foo/bar/baz", "PrepData")
+            client.stream_create("foo/bar/baz", "float32_8")
         with assert_raises(ClientError):
-            client.stream_create("/foo", "PrepData")
+            client.stream_create("/foo", "float32_8")
         # Bad layout type
         with assert_raises(ClientError):
             client.stream_create("/newton/prep", "NoSuchLayout")
 
+        # Bad method types
+        with assert_raises(ClientError):
+            client.http.put("/stream/list","")
+        # Try a bunch of times to make sure the request body is getting consumed
+        for x in range(10):
+            with assert_raises(ClientError):
+                client.http.post("/stream/list")
+        client = nilmdb.client.Client(url = testurl)
+
         # Create three streams
-        client.stream_create("/newton/prep", "PrepData")
+        client.stream_create("/newton/prep", "float32_8")
-        client.stream_create("/newton/raw", "RawData")
+        client.stream_create("/newton/raw", "uint16_6")
-        client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
+        client.stream_create("/newton/zzz/rawnotch", "uint16_9")
 
         # Verify we got 3 streams
-        eq_(client.stream_list(), [ ["/newton/prep", "PrepData"],
+        eq_(client.stream_list(), [ ["/newton/prep", "float32_8"],
-                                    ["/newton/raw", "RawData"],
+                                    ["/newton/raw", "uint16_6"],
-                                    ["/newton/zzz/rawnotch", "RawNotchedData"]
+                                    ["/newton/zzz/rawnotch", "uint16_9"]
                                     ])
         # Match just one type or one path
-        eq_(client.stream_list(layout="RawData"), [ ["/newton/raw", "RawData"] ])
+        eq_(client.stream_list(layout="uint16_6"),
-        eq_(client.stream_list(path="/newton/raw"), [ ["/newton/raw", "RawData"] ])
+            [ ["/newton/raw", "uint16_6"] ])
+        eq_(client.stream_list(path="/newton/raw"),
+            [ ["/newton/raw", "uint16_6"] ])
 
         # Try messing with resource limits to trigger errors and get
         # more coverage.  Here, make it so we can only create files 1
@@ -111,12 +128,13 @@ class TestClient(object):
         limit = resource.getrlimit(resource.RLIMIT_FSIZE)
         resource.setrlimit(resource.RLIMIT_FSIZE, (1, limit[1]))
         with assert_raises(ServerError) as e:
-            client.stream_create("/newton/hello", "RawData")
+            client.stream_create("/newton/hello", "uint16_6")
         resource.setrlimit(resource.RLIMIT_FSIZE, limit)
 
+        client.close()
 
-    def test_client_3_metadata(self):
+    def test_client_03_metadata(self):
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
 
         # Set / get metadata
         eq_(client.stream_get_metadata("/newton/prep"), {})
@@ -131,9 +149,10 @@ class TestClient(object):
         client.stream_update_metadata("/newton/raw", meta3)
         eq_(client.stream_get_metadata("/newton/prep"), meta1)
         eq_(client.stream_get_metadata("/newton/raw"), meta1)
-        eq_(client.stream_get_metadata("/newton/raw", [ "description" ] ), meta2)
+        eq_(client.stream_get_metadata("/newton/raw",
-        eq_(client.stream_get_metadata("/newton/raw", [ "description",
+                                       [ "description" ] ), meta2)
-                                                        "v_scale" ] ), meta1)
+        eq_(client.stream_get_metadata("/newton/raw",
+                                       [ "description", "v_scale" ] ), meta1)
 
         # missing key
         eq_(client.stream_get_metadata("/newton/raw", "descr"),
@@ -147,14 +166,28 @@ class TestClient(object):
         with assert_raises(ClientError):
             client.stream_update_metadata("/newton/prep", [1,2,3])
 
-    def test_client_4_insert(self):
+        # test wrong types (dict of non-strings)
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        # numbers are OK; they'll get converted to strings
+        client.stream_set_metadata("/newton/prep", { "hello": 1234 })
+        # anything else is not
+        with assert_raises(ClientError):
+            client.stream_set_metadata("/newton/prep", { "world": { 1: 2 } })
+        with assert_raises(ClientError):
+            client.stream_set_metadata("/newton/prep", { "world": [ 1, 2 ] })
+
+        client.close()
+
+    def test_client_04_insert(self):
+        client = nilmdb.client.Client(url = testurl)
+
+        # Limit _max_data to 1 MB, since our test file is 1.5 MB
+        old_max_data = nilmdb.client.client.StreamInserter._max_data
+        nilmdb.client.client.StreamInserter._max_data = 1 * 1024 * 1024
 
         datetime_tz.localtz_set("America/New_York")
 
         testfile = "tests/data/prep-20120323T1000"
-        start = datetime_tz.datetime_tz.smartparse("20120323T1000")
+        start = nilmdb.utils.time.parse_time("20120323T1000")
-        start = start.totimestamp()
         rate = 120
 
         # First try a nonexistent path
@@ -169,7 +202,8 @@ class TestClient(object):
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
         in_("400 Bad Request", str(e.exception))
-        in_("timestamp is not monotonically increasing", str(e.exception))
+        in2_("timestamp is not monotonically increasing",
+             "start must precede end", str(e.exception))
 
         # Now try empty data (no server request made)
         empty = cStringIO.StringIO("")
@@ -177,43 +211,80 @@ class TestClient(object):
         result = client.stream_insert("/newton/prep", data)
         eq_(result, None)
 
-        # Try forcing a server request with empty data
+        # It's OK to insert an empty interval
+        client.http.put("stream/insert", "", { "path": "/newton/prep",
+                                               "start": 1, "end": 2 })
+        eq_(list(client.stream_intervals("/newton/prep")), [[1, 2]])
+        client.stream_remove("/newton/prep")
+        eq_(list(client.stream_intervals("/newton/prep")), [])
+
+        # Timestamps can be negative too
+        client.http.put("stream/insert", "", { "path": "/newton/prep",
+                                               "start": -2, "end": -1 })
+        eq_(list(client.stream_intervals("/newton/prep")), [[-2, -1]])
+        client.stream_remove("/newton/prep")
+        eq_(list(client.stream_intervals("/newton/prep")), [])
+
+        # Intervals that end at zero shouldn't be any different
+        client.http.put("stream/insert", "", { "path": "/newton/prep",
+                                               "start": -1, "end": 0 })
+        eq_(list(client.stream_intervals("/newton/prep")), [[-1, 0]])
+        client.stream_remove("/newton/prep")
+        eq_(list(client.stream_intervals("/newton/prep")), [])
+
+        # Try forcing a server request with equal start and end
         with assert_raises(ClientError) as e:
             client.http.put("stream/insert", "", { "path": "/newton/prep",
                                                    "start": 0, "end": 0 })
         in_("400 Bad Request", str(e.exception))
-        in_("no data provided", str(e.exception))
+        in_("start must precede end", str(e.exception))
+
+        # Good content type
+        with assert_raises(ClientError) as e:
+            client.http.put("stream/insert", "",
+                            { "path": "xxxx", "start": 0, "end": 1,
+                              "binary": 1 },
+                            binary = True)
+        in_("No such stream", str(e.exception))
+
+        # Bad content type
+        with assert_raises(ClientError) as e:
+            client.http.put("stream/insert", "",
+                            { "path": "xxxx", "start": 0, "end": 1,
+                              "binary": 1 },
+                            binary = False)
+        in_("Content type must be application/octet-stream", str(e.exception))
 
         # Specify start/end (starts too late)
         data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
-                                          start + 5, start + 120)
+                                          start + 5000000, start + 120000000)
         in_("400 Bad Request", str(e.exception))
-        in_("Data timestamp 1332511200.0 < start time 1332511205.0",
+        in_("Data timestamp 1332511200000000 < start time 1332511205000000",
             str(e.exception))
 
         # Specify start/end (ends too early)
         data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
-                                          start, start + 1)
+                                          start, start + 1000000)
         in_("400 Bad Request", str(e.exception))
         # Client chunks the input, so the exact timestamp here might change
         # if the chunk positions change.
-        in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
+        assert(re.search("Data timestamp 13325[0-9]+ "
-            str(e.exception))
+                         ">= end time 1332511201000000", str(e.exception))
+               is not None)
 
         # Now do the real load
         data = timestamper.TimestamperRate(testfile, start, 120)
         result = client.stream_insert("/newton/prep", data,
-                                      start, start + 119.999777)
+                                      start, start + 119999777)
-        eq_(result, "ok")
 
         # Verify the intervals.  Should be just one, even if the data
         # was inserted in chunks, due to nilmdb interval concatenation.
         intervals = list(client.stream_intervals("/newton/prep"))
-        eq_(intervals, [[start, start + 119.999777]])
+        eq_(intervals, [[start, start + 119999777]])
 
         # Try some overlapping data -- just insert it again
         data = timestamper.TimestamperRate(testfile, start, 120)
@@ -222,57 +293,63 @@ class TestClient(object):
         in_("400 Bad Request", str(e.exception))
         in_("verlap", str(e.exception))
 
-    def test_client_5_extractremove(self):
+        nilmdb.client.client.StreamInserter._max_data = old_max_data
-        # Misc tests for extract and remove.  Most of them are in test_cmdline.
+        client.close()
-        client = nilmdb.Client(url = "http://localhost:12380/")
 
-        for x in client.stream_extract("/newton/prep", 123, 123):
+    def test_client_05_extractremove(self):
+        # Misc tests for extract and remove.  Most of them are in test_cmdline.
+        client = nilmdb.client.Client(url = testurl)
+
+        for x in client.stream_extract("/newton/prep",
+                                       999123000000, 999124000000):
             raise AssertionError("shouldn't be any data for this request")
 
         with assert_raises(ClientError) as e:
-            client.stream_remove("/newton/prep", 123, 120)
+            client.stream_remove("/newton/prep", 123000000, 120000000)
 
-    def test_client_6_generators(self):
+        # Test count
+        eq_(client.stream_count("/newton/prep"), 14400)
+
+        # Test binary output
+        with assert_raises(ClientError) as e:
+            list(client.stream_extract("/newton/prep",
+                                       markup = True, binary = True))
+        with assert_raises(ClientError) as e:
+            list(client.stream_extract("/newton/prep",
+                                       count = True, binary = True))
+        data = "".join(client.stream_extract("/newton/prep", binary = True))
+        # Quick check using struct
+        unpacker = struct.Struct("<qffffffff")
+        out = []
+        for i in range(14400):
+            out.append(unpacker.unpack_from(data, i * unpacker.size))
+        eq_(out[0], (1332511200000000, 266568.0, 224029.0, 5161.39990234375,
+                     2525.169921875, 8350.83984375, 3724.699951171875,
+                     1355.3399658203125, 2039.0))
+
+        client.close()
+
+    def test_client_06_generators(self):
         # A lot of the client functionality is already tested by test_cmdline,
         # but this gets a bit more coverage that cmdline misses.
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
 
         # Trigger a client error in generator
-        start = datetime_tz.datetime_tz.smartparse("20120323T2000")
+        start = nilmdb.utils.time.parse_time("20120323T2000")
-        end = datetime_tz.datetime_tz.smartparse("20120323T1000")
+        end = nilmdb.utils.time.parse_time("20120323T1000")
         for function in [ client.stream_intervals, client.stream_extract ]:
             with assert_raises(ClientError) as e:
-                function("/newton/prep",
+                function("/newton/prep", start, end).next()
-                         start.totimestamp(),
-                         end.totimestamp()).next()
             in_("400 Bad Request", str(e.exception))
-            in_("end before start", str(e.exception))
+            in_("start must precede end", str(e.exception))
 
         # Trigger a curl error in generator
         with assert_raises(ServerError) as e:
-            client.http.get_gen("http://nosuchurl/").next()
+            client.http.get_gen("http://nosuchurl.example.com./").next()
 
         # Trigger a curl error in generator
         with assert_raises(ServerError) as e:
-            client.http.get_gen("http://nosuchurl/").next()
+            client.http.get_gen("http://nosuchurl.example.com./").next()
-
-        # Check non-json version of string output
-        eq_(json.loads(client.http.get("/stream/list",retjson=False)),
-            client.http.get("/stream/list",retjson=True))
-
-        # Check non-json version of generator output
-        for (a, b) in itertools.izip(
-            client.http.get_gen("/stream/list",retjson=False),
-            client.http.get_gen("/stream/list",retjson=True)):
-            eq_(json.loads(a), b)
-
-        # Check PUT with generator out
-        with assert_raises(ClientError) as e:
-            client.http.put_gen("stream/insert", "",
-                                { "path": "/newton/prep",
-                                  "start": 0, "end": 0 }).next()
-        in_("400 Bad Request", str(e.exception))
-        in_("no data provided", str(e.exception))
 
         # Check 404 for missing streams
         for function in [ client.stream_intervals, client.stream_extract ]:
@@ -281,78 +358,365 @@ class TestClient(object):
             in_("404 Not Found", str(e.exception))
             in_("No such stream", str(e.exception))
 
-    def test_client_7_headers(self):
+        client.close()
+
+    def test_client_07_headers(self):
         # Make sure that /stream/intervals and /stream/extract
         # properly return streaming, chunked, text/plain response.
         # Pokes around in client.http internals a bit to look at the
         # response headers.
 
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
         http = client.http
 
-        # Use a warning rather than returning a test failure, so that we can
+        # Use a warning rather than returning a test failure for the
-        # still disable chunked responses for debugging.
+        # transfer-encoding, so that we can still disable chunked
+        # responses for debugging.
+
+        def headers():
+            h = ""
+            for (k, v) in http._last_response.headers.items():
+                h += k + ": " + v + "\n"
+            return h.lower()
 
         # Intervals
-        x = http.get("stream/intervals", { "path": "/newton/prep" },
+        x = http.get("stream/intervals", { "path": "/newton/prep" })
-                            retjson=False)
+        if "transfer-encoding: chunked" not in headers():
-        lines_(x, 1)
-        if "Transfer-Encoding: chunked" not in http._headers:
             warnings.warn("Non-chunked HTTP response for /stream/intervals")
-        if "Content-Type: text/plain;charset=utf-8" not in http._headers:
+        if "content-type: application/x-json-stream" not in headers():
-            raise AssertionError("/stream/intervals is not text/plain:\n" +
+            raise AssertionError("/stream/intervals content type "
-                                 http._headers)
+                                 "is not application/x-json-stream:\n" +
+                                 headers())
 
         # Extract
         x = http.get("stream/extract",
                             { "path": "/newton/prep",
                               "start": "123",
-                              "end": "123" }, retjson=False)
+                              "end": "124" })
-        if "Transfer-Encoding: chunked" not in http._headers:
+        if "transfer-encoding: chunked" not in headers():
             warnings.warn("Non-chunked HTTP response for /stream/extract")
-        if "Content-Type: text/plain;charset=utf-8" not in http._headers:
+        if "content-type: text/plain;charset=utf-8" not in headers():
             raise AssertionError("/stream/extract is not text/plain:\n" +
-                                 http._headers)
+                                 headers())
 
-        # Make sure Access-Control-Allow-Origin gets set
+        x = http.get("stream/extract",
-        if "Access-Control-Allow-Origin: " not in http._headers:
+                            { "path": "/newton/prep",
-            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                              "start": "123",
-                                 "header in /stream/extract response:\n" +
+                              "end": "124",
-                                 http._headers)
+                              "binary": "1" })
+        if "transfer-encoding: chunked" not in headers():
+            warnings.warn("Non-chunked HTTP response for /stream/extract")
+        if "content-type: application/octet-stream" not in headers():
+            raise AssertionError("/stream/extract is not binary:\n" +
+                                 headers())
 
-    def test_client_8_unicode(self):
+        client.close()
-        # Basic Unicode tests
-        client = nilmdb.Client(url = "http://localhost:12380/")
 
-        # Delete streams that exist
+    def test_client_08_unicode(self):
-        for stream in client.stream_list():
+        # Try both with and without posting JSON
-            client.stream_destroy(stream[0])
+        for post_json in (False, True):
+            # Basic Unicode tests
+            client = nilmdb.client.Client(url = testurl, post_json = post_json)
 
-        # Database is empty
+            # Delete streams that exist
-        eq_(client.stream_list(), [])
+            for stream in client.stream_list():
+                client.stream_remove(stream[0])
+                client.stream_destroy(stream[0])
 
-        # Create Unicode stream, match it
+            # Database is empty
-        raw = [ u"/düsseldorf/raw", u"uint16_6" ]
+            eq_(client.stream_list(), [])
-        prep = [ u"/düsseldorf/prep", u"uint16_6" ]
-        client.stream_create(*raw)
-        eq_(client.stream_list(), [raw])
-        eq_(client.stream_list(layout=raw[1]), [raw])
-        eq_(client.stream_list(path=raw[0]), [raw])
-        client.stream_create(*prep)
-        eq_(client.stream_list(), [prep, raw])
 
-        # Set / get metadata with Unicode keys and values
+            # Create Unicode stream, match it
-        eq_(client.stream_get_metadata(raw[0]), {})
+            raw = [ u"/düsseldorf/raw", u"uint16_6" ]
-        eq_(client.stream_get_metadata(prep[0]), {})
+            prep = [ u"/düsseldorf/prep", u"uint16_6" ]
-        meta1 = { u"alpha": u"α",
+            client.stream_create(*raw)
-                  u"β": u"beta" }
+            eq_(client.stream_list(), [raw])
-        meta2 = { u"alpha": u"α" }
+            eq_(client.stream_list(layout=raw[1]), [raw])
-        meta3 = { u"β": u"beta" }
+            eq_(client.stream_list(path=raw[0]), [raw])
-        client.stream_set_metadata(prep[0], meta1)
+            client.stream_create(*prep)
-        client.stream_update_metadata(prep[0], {})
+            eq_(client.stream_list(), [prep, raw])
-        client.stream_update_metadata(raw[0], meta2)
+
-        client.stream_update_metadata(raw[0], meta3)
+            # Set / get metadata with Unicode keys and values
-        eq_(client.stream_get_metadata(prep[0]), meta1)
+            eq_(client.stream_get_metadata(raw[0]), {})
-        eq_(client.stream_get_metadata(raw[0]), meta1)
+            eq_(client.stream_get_metadata(prep[0]), {})
-        eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
+            meta1 = { u"alpha": u"α",
-        eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
+                      u"β": u"beta" }
+            meta2 = { u"alpha": u"α" }
+            meta3 = { u"β": u"beta" }
+            client.stream_set_metadata(prep[0], meta1)
+            client.stream_update_metadata(prep[0], {})
+            client.stream_update_metadata(raw[0], meta2)
+            client.stream_update_metadata(raw[0], meta3)
+            eq_(client.stream_get_metadata(prep[0]), meta1)
+            eq_(client.stream_get_metadata(raw[0]), meta1)
+            eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
+            eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
+
+            client.close()
+
+    def test_client_09_closing(self):
+        # Make sure we actually close sockets correctly.  New
+        # connections will block for a while if they're not, since the
+        # server will stop accepting new connections.
+        for test in [1, 2]:
+            start = time.time()
+            for i in range(50):
+                if time.time() - start > 15:
+                    raise AssertionError("Connections seem to be blocking... "
+                                         "probably not closing properly.")
+                if test == 1:
+                    # explicit close
+                    client = nilmdb.client.Client(url = testurl)
+                    with assert_raises(ClientError) as e:
+                        client.stream_remove("/newton/prep", 123, 120)
+                    client.close() # remove this to see the failure
+                elif test == 2:
+                    # use the context manager
+                    with nilmdb.client.Client(url = testurl) as c:
+                        with assert_raises(ClientError) as e:
+                            c.stream_remove("/newton/prep", 123, 120)
+
+    def test_client_10_context(self):
+        # Test using the client's stream insertion context manager to
+        # insert data.
+        client = nilmdb.client.Client(testurl)
+
+        client.stream_create("/context/test", "uint16_1")
+        with client.stream_insert_context("/context/test") as ctx:
+            # override _max_data to trigger frequent server updates
+            ctx._max_data = 15
+
+            ctx.insert("1000 1\n")
+
+            ctx.insert("1010 ")
+            ctx.insert("1\n1020 1")
+            ctx.insert("")
+            ctx.insert("\n1030 1\n")
+
+            ctx.insert("1040 1\n")
+            ctx.insert("# hello\n")
+            ctx.insert("   # hello\n")
+            ctx.insert("  1050 1\n")
+            ctx.finalize()
+
+            ctx.insert("1070 1\n")
+            ctx.update_end(1080)
+            ctx.finalize()
+            ctx.update_start(1090)
+            ctx.insert("1100 1\n")
+            ctx.insert("1110 1\n")
+            ctx.send()
+            ctx.insert("1120 1\n")
+            ctx.insert("1130 1\n")
+            ctx.insert("1140 1\n")
+            ctx.update_end(1160)
+            ctx.insert("1150 1\n")
+            ctx.update_end(1170)
+            ctx.insert("1160 1\n")
+            ctx.update_end(1180)
+            ctx.insert("1170 1" +
+                       " # this is super long" * 100 +
+                       "\n")
+            ctx.finalize()
+            ctx.insert("# this is super long" * 100)
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test",
+                                              1000, 2000) as ctx:
+                ctx.insert("1180 1\n")
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test",
+                                              2000, 3000) as ctx:
+                ctx.insert("1180 1\n")
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test") as ctx:
+                ctx.insert("bogus data\n")
+
+        with client.stream_insert_context("/context/test", 2000, 3000) as ctx:
+            # make sure our override wasn't permanent
+            ne_(ctx._max_data, 15)
+            ctx.insert("2250 1\n")
+            ctx.finalize()
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test",
+                                              3000, 4000) as ctx:
+                ctx.insert("3010 1\n")
+                ctx.insert("3020 2\n")
+                ctx.insert("3030 3\n")
+                ctx.insert("3040 4\n")
+                ctx.insert("3040 4\n") # non-monotonic after a few lines
+                ctx.finalize()
+
+        eq_(list(client.stream_intervals("/context/test")),
+            [ [ 1000, 1051 ],
+              [ 1070, 1080 ],
+              [ 1090, 1180 ],
+              [ 2000, 3000 ] ])
+
+        # destroy stream (try without removing data first)
+        with assert_raises(ClientError):
+            client.stream_destroy("/context/test")
+        client.stream_remove("/context/test")
+        client.stream_destroy("/context/test")
+        client.close()
+
+    def test_client_11_emptyintervals(self):
+        # Empty intervals are ok!  If recording detection events
+        # by inserting rows into the database, we want to be able to
+        # have an interval where no events occurred.  Test them here.
+        client = nilmdb.client.Client(testurl)
+        client.stream_create("/empty/test", "uint16_1")
+
+        def info():
+            result = []
+            for interval in list(client.stream_intervals("/empty/test")):
+                result.append((client.stream_count("/empty/test", *interval),
+                               interval))
+            return result
+
+        eq_(info(), [])
+
+        # Insert a region with just a few points
+        with client.stream_insert_context("/empty/test") as ctx:
+            ctx.update_start(100)
+            ctx.insert("140 1\n")
+            ctx.insert("150 1\n")
+            ctx.insert("160 1\n")
+            ctx.update_end(200)
+            ctx.finalize()
+
+        eq_(info(), [(3, [100, 200])])
+
+        # Delete chunk, which will leave one data point and two intervals
+        client.stream_remove("/empty/test", 145, 175)
+        eq_(info(), [(1, [100, 145]),
+                     (0, [175, 200])])
+
+        # Try also creating a completely empty interval from scratch,
+        # in a few different ways.
+        client.stream_insert("/empty/test", "", 300, 350)
+        client.stream_insert("/empty/test", [], 400, 450)
+        with client.stream_insert_context("/empty/test", 500, 550):
+            pass
+
+        # If enough timestamps aren't provided, empty streams won't be created.
+        client.stream_insert("/empty/test", [])
+        with client.stream_insert_context("/empty/test"):
+            pass
+        client.stream_insert("/empty/test", [], start = 600)
+        with client.stream_insert_context("/empty/test", start = 700):
+            pass
+        client.stream_insert("/empty/test", [], end = 850)
+        with client.stream_insert_context("/empty/test", end = 950):
+            pass
+
+        # Try various things that might cause problems
+        with client.stream_insert_context("/empty/test", 1000, 1050):
+            ctx.finalize() # inserts [1000, 1050]
+            ctx.finalize() # nothing
+            ctx.finalize() # nothing
+            ctx.insert("1100 1\n")
+            ctx.finalize() # inserts [1100, 1101]
+            ctx.update_start(1199)
+            ctx.insert("1200 1\n")
+            ctx.update_end(1250)
+            ctx.finalize() # inserts [1199, 1250]
+            ctx.update_start(1299)
+            ctx.finalize() # nothing
+            ctx.update_end(1350)
+            ctx.finalize() # nothing
+            ctx.update_start(1400)
+            ctx.insert("# nothing!\n")
+            ctx.update_end(1450)
+            ctx.finalize()
+            ctx.update_start(1500)
+            ctx.insert("# nothing!")
+            ctx.update_end(1550)
+            ctx.finalize()
+            ctx.insert("# nothing!\n" * 10)
+            ctx.finalize()
+            # implicit last finalize inserts [1400, 1450]
+
+        # Check everything
+        eq_(info(), [(1, [100, 145]),
+                     (0, [175, 200]),
+                     (0, [300, 350]),
+                     (0, [400, 450]),
+                     (0, [500, 550]),
+                     (0, [1000, 1050]),
+                     (1, [1100, 1101]),
+                     (1, [1199, 1250]),
+                     (0, [1400, 1450]),
+                     (0, [1500, 1550]),
+                     ])
+
+        # Clean up
+        client.stream_remove("/empty/test")
+        client.stream_destroy("/empty/test")
+        client.close()
+
+    def test_client_12_persistent(self):
+        # Check that connections are persistent when they should be.
+        # This is pretty hard to test; we have to poke deep into
+        # the Requests library.
+        with nilmdb.client.Client(url = testurl) as c:
+            def connections():
+                try:
+                    poolmanager = c.http._last_response.connection.poolmanager
+                    pool = poolmanager.pools[('http','localhost',32180)]
+                    return (pool.num_connections, pool.num_requests)
+                except Exception:
+                    raise SkipTest("can't get connection info")
+
+            # First request makes a connection
+            c.stream_create("/persist/test", "uint16_1")
+            eq_(connections(), (1, 1))
+
+            # Non-generator
+            c.stream_list("/persist/test")
+            eq_(connections(), (1, 2))
+            c.stream_list("/persist/test")
+            eq_(connections(), (1, 3))
+
+            # Generators
+            for x in c.stream_intervals("/persist/test"):
+                pass
+            eq_(connections(), (1, 4))
+            for x in c.stream_intervals("/persist/test"):
+                pass
+            eq_(connections(), (1, 5))
+
+            # Clean up
+            c.stream_remove("/persist/test")
+            c.stream_destroy("/persist/test")
+            eq_(connections(), (1, 7))
+
+    def test_client_13_timestamp_rounding(self):
+        # Test potentially bad timestamps (due to floating point
+        # roundoff etc).  The server will round floating point values
+        # to the nearest int.
+        client = nilmdb.client.Client(testurl)
+
+        client.stream_create("/rounding/test", "uint16_1")
+        with client.stream_insert_context("/rounding/test",
+                                          100000000, 200000000.1) as ctx:
+            ctx.insert("100000000.1 1\n")
+            ctx.insert("150000000.00003 1\n")
+            ctx.insert("199999999.4 1\n")
+        eq_(list(client.stream_intervals("/rounding/test")),
+            [ [ 100000000, 200000000 ] ])
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/rounding/test",
+                                              200000000, 300000000) as ctx:
+                ctx.insert("200000000 1\n")
+                ctx.insert("250000000 1\n")
+                # Server will round this and give an error on finalize()
+                ctx.insert("299999999.99 1\n")
+
+        client.stream_remove("/rounding/test")
+        client.stream_destroy("/rounding/test")
+        client.close()

tests/test_cmdline.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
-import nilmdb
+import nilmdb.server
+
 from nilmdb.utils.printf import *
 import nilmdb.cmdline
 from nilmdb.utils import datetime_tz
@@ -11,29 +12,27 @@ from nose.tools import assert_raises
 import itertools
 import os
 import re
-import shutil
 import sys
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
 import StringIO
 import shlex
+import warnings
 
 from testutil.helpers import *
 
 testdb = "tests/cmdline-testdb"
 
-def server_start(max_results = None, bulkdata_args = {}):
+def server_start(max_results = None, max_removals = None, bulkdata_args = {}):
     global test_server, test_db
     # Start web app on a custom port
-    test_db = nilmdb.NilmDB(testdb, sync = False,
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(
-                            max_results = max_results,
+        testdb,
-                            bulkdata_args = bulkdata_args)
+        max_results = max_results,
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
+        max_removals = max_removals,
-                                port = 12380, stoppable = False,
+        bulkdata_args = bulkdata_args)
-                                fast_shutdown = True,
+    test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
-                                force_traceback = False)
+                                       port = 32180, stoppable = False,
+                                       fast_shutdown = True,
+                                       force_traceback = False)
     test_server.start(blocking = False)
 
 def server_stop():
@@ -63,6 +62,7 @@ class TestCmdline(object):
         passing the given input.  Returns a tuple with the output and
         exit code"""
         # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
+        os.environ['NILMDB_URL'] = "http://localhost:32180/"
         class stdio_wrapper:
             def __init__(self, stdin, stdout, stderr):
                 self.io = (stdin, stdout, stderr)
@@ -128,8 +128,17 @@ class TestCmdline(object):
         with open(file) as f:
             contents = f.read()
             if contents != self.captured:
-                print contents[1:1000] + "\n"
+                print "--- reference file (first 1000 bytes):\n"
-                print self.captured[1:1000] + "\n"
+                print contents[0:1000] + "\n"
+                print "--- captured data (first 1000 bytes):\n"
+                print self.captured[0:1000] + "\n"
+                zipped = itertools.izip_longest(contents, self.captured)
+                for (n, (a, b)) in enumerate(zipped):
+                    if a != b:
+                        print "--- first difference is at offset", n
+                        print "--- reference:", repr(a)
+                        print "---  captured:", repr(b)
+                        break
                 raise AssertionError("captured data doesn't match " + file)
 
     def matchfilecount(self, file):
@@ -162,18 +171,18 @@ class TestCmdline(object):
 
         # try some URL constructions
         self.fail("--url http://nosuchurl/ info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
+        self.contain("error connecting to server")
 
         self.fail("--url nosuchurl info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
+        self.contain("error connecting to server")
 
         self.fail("-u nosuchurl/foo info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
+        self.contain("error connecting to server")
 
-        self.fail("-u localhost:0 info")
+        self.fail("-u localhost:1 info")
-        self.contain("couldn't connect to host")
+        self.contain("error connecting to server")
 
-        self.ok("-u localhost:12380 info")
+        self.ok("-u localhost:32180 info")
         self.ok("info")
 
         # Duplicated arguments should fail, but this isn't implemented
@@ -191,14 +200,55 @@ class TestCmdline(object):
             self.fail("extract --start 2000-01-01 --start 2001-01-02")
             self.contain("duplicated argument")
 
-    def test_02_info(self):
+        # Verify that "help command" and "command --help" are identical
+        # for all commands.
+        self.fail("")
+        m = re.search(r"{(.*)}", self.captured)
+        for command in [""] + m.group(1).split(','):
+            self.ok(command + " --help")
+            cap1 = self.captured
+            self.ok("help " + command)
+            cap2 = self.captured
+            self.ok("help " + command + " asdf --url --zxcv -")
+            cap3 = self.captured
+            eq_(cap1, cap2)
+            eq_(cap2, cap3)
+
+    def test_02_parsetime(self):
+        os.environ['TZ'] = "America/New_York"
+        test = datetime_tz.datetime_tz.now()
+        u2ts = nilmdb.utils.time.unix_to_timestamp
+        parse_time = nilmdb.utils.time.parse_time
+        eq_(parse_time(str(test)), u2ts(test.totimestamp()))
+        test = u2ts(datetime_tz.datetime_tz.smartparse("20120405 1400-0400").
+                    totimestamp())
+        eq_(parse_time("hi there 20120405 1400-0400 testing! 123"), test)
+        eq_(parse_time("20120405 1800 UTC"), test)
+        eq_(parse_time("20120405 1400-0400 UTC"), test)
+        for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
+            with assert_raises(ValueError):
+                x = parse_time(badtime)
+        x = parse_time("now")
+        eq_(parse_time("snapshot-20120405-140000.raw.gz"), test)
+        eq_(parse_time("prep-20120405T1400"), test)
+        eq_(parse_time("1333648800.0"), test)
+        eq_(parse_time("1333648800000000"), test)
+        eq_(parse_time("@1333648800000000"), test)
+        eq_(parse_time("min"), nilmdb.utils.time.min_timestamp)
+        eq_(parse_time("max"), nilmdb.utils.time.max_timestamp)
+        with assert_raises(ValueError):
+            parse_time("@hashtag12345")
+
+    def test_03_info(self):
         self.ok("info")
-        self.contain("Server URL: http://localhost:12380/")
+        self.contain("Server URL: http://localhost:32180/")
+        self.contain("Client version: " + nilmdb.__version__)
         self.contain("Server version: " + test_server.version)
         self.contain("Server database path")
         self.contain("Server database size")
+        self.contain("Server database free space")
 
-    def test_03_createlist(self):
+    def test_04_createlist(self):
         # Basic stream tests, like those in test_client.
 
         # No streams
@@ -206,11 +256,20 @@ class TestCmdline(object):
         self.match("")
 
         # Bad paths
-        self.fail("create foo/bar/baz PrepData")
+        self.fail("create foo/bar/baz float32_8")
         self.contain("paths must start with /")
 
-        self.fail("create /foo PrepData")
+        self.fail("create /foo float32_8")
         self.contain("invalid path")
+        self.fail("create /newton/prep/ float32_8")
+        self.contain("invalid path")
+
+        self.fail("create /newton/_format/prep float32_8")
+        self.contain("path name is invalid")
+        self.fail("create /_format/newton/prep float32_8")
+        self.contain("path name is invalid")
+        self.fail("create /newton/prep/_format float32_8")
+        self.contain("path name is invalid")
 
         # Bad layout type
         self.fail("create /newton/prep NoSuchLayout")
@@ -221,60 +280,64 @@ class TestCmdline(object):
         self.contain("no such layout")
 
         # Create a few streams
-        self.ok("create /newton/zzz/rawnotch RawNotchedData")
+        self.ok("create /newton/zzz/rawnotch uint16_9")
-        self.ok("create /newton/prep PrepData")
+        self.ok("create /newton/prep float32_8")
-        self.ok("create /newton/raw RawData")
+        self.ok("create /newton/raw uint16_6")
+
+        # Create a stream that already exists
+        self.fail("create /newton/raw uint16_6")
+        self.contain("stream already exists at this path")
 
         # Should not be able to create a stream with another stream as
         # its parent
-        self.fail("create /newton/prep/blah PrepData")
+        self.fail("create /newton/prep/blah float32_8")
         self.contain("path is subdir of existing node")
 
         # Should not be able to create a stream at a location that
         # has other nodes as children
-        self.fail("create /newton/zzz PrepData")
+        self.fail("create /newton/zzz float32_8")
         self.contain("subdirs of this path already exist")
 
         # Verify we got those 3 streams and they're returned in
         # alphabetical order.
         self.ok("list")
-        self.match("/newton/prep PrepData\n"
+        self.match("/newton/prep float32_8\n"
-                   "/newton/raw RawData\n"
+                   "/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         # Match just one type or one path.  Also check
         # that --path is optional
         self.ok("list --path /newton/raw")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         self.ok("list /newton/raw")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         self.fail("list -p /newton/raw /newton/raw")
         self.contain("too many paths")
 
-        self.ok("list --layout RawData")
+        self.ok("list --layout uint16_6")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         # Wildcard matches
-        self.ok("list --layout Raw*")
+        self.ok("list --layout uint16*")
-        self.match("/newton/raw RawData\n"
+        self.match("/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list --path *zzz* --layout Raw*")
+        self.ok("list --path *zzz* --layout uint16*")
-        self.match("/newton/zzz/rawnotch RawNotchedData\n")
+        self.match("/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list *zzz* --layout Raw*")
+        self.ok("list *zzz* --layout uint16*")
-        self.match("/newton/zzz/rawnotch RawNotchedData\n")
+        self.match("/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list --path *zzz* --layout Prep*")
+        self.ok("list --path *zzz* --layout float32*")
         self.match("")
 
         # reversed range
         self.fail("list /newton/prep --start 2020-01-01 --end 2000-01-01")
-        self.contain("start is after end")
+        self.contain("start must precede end")
 
-    def test_04_metadata(self):
+    def test_05_metadata(self):
         # Set / get metadata
         self.fail("metadata")
         self.fail("metadata --get")
@@ -306,6 +369,8 @@ class TestCmdline(object):
         self.contain("No stream at path")
         self.fail("metadata /newton/nosuchstream --set foo=bar")
         self.contain("No stream at path")
+        self.fail("metadata /newton/nosuchstream --delete")
+        self.contain("No stream at path")
 
         self.ok("metadata /newton/prep")
         self.match("description=The Data\nv_scale=1.234\n")
@@ -331,62 +396,65 @@ class TestCmdline(object):
         self.fail("metadata /newton/nosuchpath")
         self.contain("No stream at path /newton/nosuchpath")
 
-    def test_05_parsetime(self):
+        self.ok("metadata /newton/prep --delete")
-        os.environ['TZ'] = "America/New_York"
+        self.ok("metadata /newton/prep --get")
-        cmd = nilmdb.cmdline.Cmdline(None)
+        self.match("")
-        test = datetime_tz.datetime_tz.now()
+        self.ok("metadata /newton/prep --set "
-        eq_(cmd.parse_time(str(test)), test)
+                "'description=The Data' "
-        test = datetime_tz.datetime_tz.smartparse("20120405 1400-0400")
+                "v_scale=1.234")
-        eq_(cmd.parse_time("hi there 20120405 1400-0400 testing! 123"), test)
+        self.ok("metadata /newton/prep --delete v_scale")
-        eq_(cmd.parse_time("20120405 1800 UTC"), test)
+        self.ok("metadata /newton/prep --get")
-        eq_(cmd.parse_time("20120405 1400-0400 UTC"), test)
+        self.match("description=The Data\n")
-        for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
+        self.ok("metadata /newton/prep --set description=")
-            with assert_raises(ValueError):
+        self.ok("metadata /newton/prep --get")
-                x = cmd.parse_time(badtime)
+        self.match("")
-        x = cmd.parse_time("now")
-        eq_(cmd.parse_time("snapshot-20120405-140000.raw.gz"), test)
-        eq_(cmd.parse_time("prep-20120405T1400"), test)
 
     def test_06_insert(self):
         self.ok("insert --help")
 
-        self.fail("insert /foo/bar baz qwer")
+        self.fail("insert -s 2000 -e 2001 /foo/bar baz")
         self.contain("error getting stream info")
 
-        self.fail("insert /newton/prep baz qwer")
+        self.fail("insert -s 2000 -e 2001 /newton/prep baz")
         self.match("error opening input file baz\n")
 
-        self.fail("insert /newton/prep")
+        self.fail("insert /newton/prep --timestamp -f -r 120")
-        self.contain("error extracting time")
+        self.contain("error extracting start time")
 
-        self.fail("insert --start 19801205 /newton/prep 1 2 3 4")
+        self.fail("insert /newton/prep --timestamp -r 120")
-        self.contain("--start can only be used with one input file")
+        self.contain("need --start or --filename")
 
         self.fail("insert /newton/prep "
                   "tests/data/prep-20120323T1000")
 
-        # insert pre-timestamped data, from stdin
-        os.environ['TZ'] = "UTC"
-        with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
-
         # insert pre-timestamped data, with bad times (non-monotonic)
         os.environ['TZ'] = "UTC"
         with open("tests/data/prep-20120323T1004-badtimes") as input:
-            self.fail("insert --none /newton/prep", input)
+            self.fail("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                      input)
             self.contain("error parsing input data")
-            self.contain("line 7:")
+            self.contain("line 7")
             self.contain("timestamp is not monotonically increasing")
 
+        # insert pre-timestamped data, from stdin
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-timestamped") as input:
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
+
         # insert data with normal timestamper from filename
         os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
+        self.ok("insert --timestamp -f --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1000 "
+                "tests/data/prep-20120323T1000")
+        self.fail("insert -t --filename /newton/prep "
+                "tests/data/prep-20120323T1002")
+        self.contain("rate is needed")
+        self.ok("insert -t --filename --rate 120 /newton/prep "
                 "tests/data/prep-20120323T1002")
 
         # overlap
         os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/prep "
+        self.fail("insert --timestamp -f --rate 120 /newton/prep "
                   "tests/data/prep-20120323T1004")
         self.contain("overlap")
 
@@ -398,24 +466,33 @@ class TestCmdline(object):
 
         # still an overlap if we specify a different start
         os.environ['TZ'] = "America/New_York"
-        self.fail("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep"
+        self.fail("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep"
                   " tests/data/prep-20120323T1004")
         self.contain("overlap")
 
         # wrong format
         os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/raw "
+        self.fail("insert -t -r 120 -f /newton/raw "
                   "tests/data/prep-20120323T1004")
         self.contain("error parsing input data")
+        self.contain("can't parse value")
+
+        # too few rows per line
+        self.ok("create /insert/test float32_20")
+        self.fail("insert -t -r 120 -f /insert/test "
+                  "tests/data/prep-20120323T1004")
+        self.contain("error parsing input data")
+        self.contain("wrong number of values")
+        self.ok("destroy /insert/test")
 
         # empty data does nothing
-        self.ok("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep "
+        self.ok("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep "
                 "/dev/null")
 
         # bad start time
-        self.fail("insert --rate 120 --start 'whatever' /newton/prep /dev/null")
+        self.fail("insert -t -r 120 --start 'whatever' /newton/prep /dev/null")
 
-    def test_07_detail(self):
+    def test_07_detail_extended(self):
         # Just count the number of lines, it's probably fine
         self.ok("list --detail")
         lines_(self.captured, 8)
@@ -442,7 +519,7 @@ class TestCmdline(object):
         self.contain("no intervals")
 
         self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'"
-                + " --end='23 Mar 2012 10:05:15.50'")
+                + " --end='23 Mar 2012 10:05:15.51'")
         lines_(self.captured, 2)
         self.contain("10:05:15.500")
 
@@ -453,12 +530,29 @@ class TestCmdline(object):
         self.ok("list --detail --path *prep --timestamp-raw "
                 "--start='23 Mar 2012 10:05:15.50'")
         lines_(self.captured, 2)
-        self.contain("[ 1332497115.5 -> 1332497159.991668 ]")
+        self.contain("[ 1332497115500000 -> 1332497160000000 ]")
 
-        self.ok("list --detail --path *prep -T "
+        # bad time
-                "--start='23 Mar 2012 10:05:15.612'")
+        self.fail("list --detail --path *prep -T --start='9332497115.612'")
+        # good time
+        self.ok("list --detail --path *prep -T --start='1332497115.612'")
         lines_(self.captured, 2)
-        self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
+        self.contain("[ 1332497115612000 -> 1332497160000000 ]")
+
+        # Check --ext output
+        self.ok("list --ext")
+        lines_(self.captured, 9)
+
+        self.ok("list -E -T")
+        c = self.contain
+        c("\n  interval extents: 1332496800000000 -> 1332497160000000\n")
+        c("\n        total data: 43200 rows, 359.983336 seconds\n")
+        c("\n  interval extents: (no data)\n")
+        c("\n        total data: 0 rows, 0.000000 seconds\n")
+
+        # Misc
+        self.fail("list --ext --start='23 Mar 2012 10:05:15.50'")
+        self.contain("--start and --end only make sense with --detail")
 
     def test_08_extract(self):
         # nonexistent stream
@@ -471,29 +565,29 @@ class TestCmdline(object):
 
         # empty ranges return error 2
         self.fail("extract -a /newton/prep " +
-                  "--start '23 Mar 2012 10:00:30' " +
+                  "--start '23 Mar 2012 20:00:30' " +
-                  "--end '23 Mar 2012 10:00:30'",
+                  "--end '23 Mar 2012 20:00:31'",
                   exitcode = 2, require_error = False)
         self.contain("no data")
         self.fail("extract -a /newton/prep " +
-                  "--start '23 Mar 2012 10:00:30.000001' " +
+                  "--start '23 Mar 2012 20:00:30.000001' " +
-                  "--end '23 Mar 2012 10:00:30.000001'",
+                  "--end '23 Mar 2012 20:00:30.000002'",
                   exitcode = 2, require_error = False)
         self.contain("no data")
         self.fail("extract -a /newton/prep " +
                   "--start '23 Mar 2022 10:00:30' " +
-                  "--end '23 Mar 2022 10:00:30'",
+                  "--end '23 Mar 2022 10:00:31'",
                   exitcode = 2, require_error = False)
         self.contain("no data")
 
         # but are ok if we're just counting results
         self.ok("extract --count /newton/prep " +
-                "--start '23 Mar 2012 10:00:30' " +
+                "--start '23 Mar 2012 20:00:30' " +
-                "--end '23 Mar 2012 10:00:30'")
+                "--end '23 Mar 2012 20:00:31'")
         self.match("0\n")
         self.ok("extract -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:30.000001' " +
+                "--start '23 Mar 2012 20:00:30.000001' " +
-                "--end '23 Mar 2012 10:00:30.000001'")
+                "--end '23 Mar 2012 20:00:30.000002'")
         self.match("0\n")
 
         # Check various dumps against stored copies of how they should appear
@@ -517,13 +611,20 @@ class TestCmdline(object):
         test(6, "10:00:30", "10:00:31", extra="-b")
         test(7, "10:00:30", "10:00:30.999", extra="-a -T")
         test(7, "10:00:30", "10:00:30.999", extra="-a --timestamp-raw")
+        test(8, "10:01:59.9", "10:02:00.1", extra="--markup")
+        test(8, "10:01:59.9", "10:02:00.1", extra="-m")
 
         # all data put in by tests
-        self.ok("extract -a /newton/prep --start 2000-01-01 --end 2020-01-01")
+        self.ok("extract -a /newton/prep --start min --end max")
         lines_(self.captured, 43204)
         self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
         self.match("43200\n")
 
+        # markup for 3 intervals, plus extra markup lines whenever we had
+        # a "restart" from the nilmdb.stream_extract function
+        self.ok("extract -m /newton/prep --start 2000-01-01 --end 2020-01-01")
+        lines_(self.captured, 43210)
+
     def test_09_truncated(self):
         # Test truncated responses by overriding the nilmdb max_results
         server_stop()
@@ -540,42 +641,42 @@ class TestCmdline(object):
         self.fail("remove /no/such/foo --start 2000-01-01 --end 2020-01-01")
         self.contain("No stream at path")
 
+        # empty or backward ranges return errors
         self.fail("remove /newton/prep --start 2020-01-01 --end 2000-01-01")
-        self.contain("start is after end")
+        self.contain("start must precede end")
 
-        # empty ranges return success, backwards ranges return error
+        self.fail("remove /newton/prep " +
-        self.ok("remove /newton/prep " +
+                  "--start '23 Mar 2012 10:00:30' " +
-                "--start '23 Mar 2012 10:00:30' " +
+                  "--end '23 Mar 2012 10:00:30'")
-                "--end '23 Mar 2012 10:00:30'")
+        self.contain("start must precede end")
-        self.match("")
+        self.fail("remove /newton/prep " +
-        self.ok("remove /newton/prep " +
+                  "--start '23 Mar 2012 10:00:30.000001' " +
-                "--start '23 Mar 2012 10:00:30.000001' " +
+                  "--end '23 Mar 2012 10:00:30.000001'")
-                "--end '23 Mar 2012 10:00:30.000001'")
+        self.contain("start must precede end")
-        self.match("")
+        self.fail("remove /newton/prep " +
-        self.ok("remove /newton/prep " +
+                  "--start '23 Mar 2022 10:00:30' " +
-                "--start '23 Mar 2022 10:00:30' " +
+                  "--end '23 Mar 2022 10:00:30'")
-                "--end '23 Mar 2022 10:00:30'")
+        self.contain("start must precede end")
-        self.match("")
 
         # Verbose
         self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:30' " +
+                "--start '23 Mar 2022 20:00:30' " +
-                "--end '23 Mar 2012 10:00:30'")
+                "--end '23 Mar 2022 20:00:31'")
         self.match("0\n")
         self.ok("remove --count /newton/prep " +
-                "--start '23 Mar 2012 10:00:30' " +
+                "--start '23 Mar 2022 20:00:30' " +
-                "--end '23 Mar 2012 10:00:30'")
+                "--end '23 Mar 2022 20:00:31'")
         self.match("0\n")
 
         # Make sure we have the data we expect
         self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:04:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:05:59.991668 +0000 ]\n")
+                   " -> Fri, 23 Mar 2012 10:06:00.000000 +0000 ]\n")
 
         # Remove various chunks of prep data and make sure
         # they're gone.
@@ -604,7 +705,7 @@ class TestCmdline(object):
 
         # See the missing chunks in list output
         self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
@@ -618,15 +719,14 @@ class TestCmdline(object):
         self.ok("remove /newton/prep --start 2000-01-01 --end 2020-01-01")
         self.match("")  # no count requested this time
         self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  (no intervals)\n")
 
         # Reinsert some data, to verify that no overlaps with deleted
         # data are reported
-        os.environ['TZ'] = "UTC"
+        for minute in ["0", "2"]:
-        self.ok("insert --rate 120 /newton/prep "
+            self.ok("insert --timestamp -f --rate 120 /newton/prep"
-                "tests/data/prep-20120323T1000 "
+                    " tests/data/prep-20120323T100" + minute)
-                "tests/data/prep-20120323T1002")
 
     def test_11_destroy(self):
         # Delete records
@@ -638,32 +738,42 @@ class TestCmdline(object):
         self.fail("destroy /no/such/stream")
         self.contain("No stream at path")
 
+        self.fail("destroy -R /no/such/stream")
+        self.contain("No stream at path")
+
         self.fail("destroy asdfasdf")
         self.contain("No stream at path")
 
         # From previous tests, we have:
         self.ok("list")
-        self.match("/newton/prep PrepData\n"
+        self.match("/newton/prep float32_8\n"
-                   "/newton/raw RawData\n"
+                   "/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         # Notice how they're not empty
         self.ok("list --detail")
         lines_(self.captured, 7)
 
-        # Delete some
+        # Fail to destroy because intervals still present
-        self.ok("destroy /newton/prep")
+        self.fail("destroy /newton/prep")
+        self.contain("all intervals must be removed")
+        self.ok("list --detail")
+        lines_(self.captured, 7)
+
+        # Destroy for real
+        self.ok("destroy -R /newton/prep")
         self.ok("list")
-        self.match("/newton/raw RawData\n"
+        self.match("/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         self.ok("destroy /newton/zzz/rawnotch")
         self.ok("list")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         self.ok("destroy /newton/raw")
-        self.ok("create /newton/raw RawData")
+        self.ok("create /newton/raw uint16_6")
-        self.ok("destroy /newton/raw")
+        # Specify --remove with no data
+        self.ok("destroy --remove /newton/raw")
         self.ok("list")
         self.match("")
 
@@ -672,7 +782,7 @@ class TestCmdline(object):
                     "/newton/raw", "/newton/asdf/qwer" ]
         for path in rebuild:
             # Create the path
-            self.ok("create " + path + " PrepData")
+            self.ok("create " + path + " float32_8")
             self.ok("list")
             self.contain(path)
             # Make sure it was created empty
@@ -710,7 +820,8 @@ class TestCmdline(object):
         self.ok("create /newton/prep float32_8")
         os.environ['TZ'] = "UTC"
         with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
 
         # Extract it
         self.ok("extract /newton/prep --start '2000-01-01' " +
@@ -737,39 +848,43 @@ class TestCmdline(object):
 
         # Now recreate the data one more time and make sure there are
         # fewer files.
-        self.ok("destroy /newton/prep")
+        self.ok("destroy --remove /newton/prep")
         self.fail("destroy /newton/prep") # already destroyed
         self.ok("create /newton/prep float32_8")
         os.environ['TZ'] = "UTC"
         with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
         nfiles = 0
         for (dirpath, dirnames, filenames) in os.walk(testdb):
             nfiles += len(filenames)
         lt_(nfiles, 50)
-        self.ok("destroy /newton/prep") # destroy again
+        self.ok("destroy -R /newton/prep") # destroy again
 
     def test_14_remove_files(self):
         # Test BulkData's ability to remove when data is split into
         # multiple files.  Should be a fairly comprehensive test of
         # remove functionality.
+        # Also limit max_removals, to cover more functionality.
         server_stop()
-        server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
+        server_start(max_removals = 4321,
+                     bulkdata_args = { "file_size" : 920, # 23 rows per file
                                        "files_per_dir" : 3 })
 
         # Insert data.  Just for fun, insert out of order
-        self.ok("create /newton/prep PrepData")
+        self.ok("create /newton/prep float32_8")
         os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
+        self.ok("insert -t --filename --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1002 "
+                "tests/data/prep-20120323T1002")
+        self.ok("insert -t --filename --rate 120 /newton/prep "
                 "tests/data/prep-20120323T1000")
 
         # Should take up about 2.8 MB here (including directory entries)
-        du_before = nilmdb.utils.diskusage.du_bytes(testdb)
+        du_before = nilmdb.utils.diskusage.du(testdb)
 
         # Make sure we have the data we expect
         self.ok("list --detail")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
@@ -805,7 +920,7 @@ class TestCmdline(object):
 
         # See the missing chunks in list output
         self.ok("list --detail")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
@@ -815,7 +930,7 @@ class TestCmdline(object):
 
         # We have 1/8 of the data that we had before, so the file size
         # should have dropped below 1/4 of what it used to be
-        du_after = nilmdb.utils.diskusage.du_bytes(testdb)
+        du_after = nilmdb.utils.diskusage.du(testdb)
         lt_(du_after, (du_before / 4))
 
         # Remove anything that came from the 10:02 data file
@@ -826,7 +941,7 @@ class TestCmdline(object):
         # With the specific file_size above, this will cause the last
         # file in the bulk data storage to be exactly file_size large,
         # so removing the data should also remove that last file.
-        self.ok("insert --rate 120 /newton/prep " +
+        self.ok("insert --timestamp -f --rate 120 /newton/prep " +
                 "tests/data/prep-20120323T1002-first19lines")
         self.ok("remove /newton/prep " +
                 "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
@@ -837,8 +952,146 @@ class TestCmdline(object):
 
         # Re-add the full 10:02 data file.  This tests adding new data once
         # we removed data near the end.
-        self.ok("insert --rate 120 /newton/prep tests/data/prep-20120323T1002")
+        self.ok("insert -t -f -r 120 /newton/prep "
+                "tests/data/prep-20120323T1002")
 
         # See if we can extract it all
         self.ok("extract /newton/prep --start 2000-01-01 --end 2020-01-01")
         lines_(self.captured, 15600)
+
+    def test_15_intervals_diff(self):
+        # Test "intervals" and "intervals --diff" command.
+        os.environ['TZ'] = "UTC"
+
+        self.ok("create /diff/1 uint8_1")
+        self.match("")
+        self.ok("intervals /diff/1")
+        self.match("")
+        self.ok("intervals /diff/1 --diff /diff/1")
+        self.match("")
+        self.ok("intervals --diff /diff/1 /diff/1")
+        self.match("")
+        self.fail("intervals /diff/2")
+        self.fail("intervals /diff/1 -d /diff/2")
+
+        self.ok("create /diff/2 uint8_1")
+        self.ok("intervals -T /diff/1 -d /diff/2")
+        self.match("")
+        self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/1 /dev/null")
+
+        self.ok("intervals /diff/1")
+        self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
+                   "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
+
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
+                   "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
+
+        self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/2 /dev/null")
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("")
+
+        self.ok("insert -s 01-01-2001 -e 01-01-2002 /diff/1 /dev/null")
+        self.ok("insert -s 01-01-2002 -e 01-01-2003 /diff/2 /dev/null")
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
+                   "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n")
+
+        self.ok("insert -s 01-01-2004 -e 01-01-2005 /diff/1 /dev/null")
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
+                   "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n"
+                   "[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
+                   "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
+
+        self.fail("intervals -s 01-01-2003 -e 01-01-2000 /diff/1 -d /diff/2")
+        self.ok("intervals -s 01-01-2003 -e 01-01-2008 /diff/1 -d /diff/2")
+        self.match("[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
+                   "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
+
+        self.ok("destroy -R /diff/1")
+        self.ok("destroy -R /diff/2")
+
+    def test_16_rename(self):
+        # Test renaming.  Force file size smaller so we get more files
+        server_stop()
+        recursive_unlink(testdb)
+        server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
+                                       "files_per_dir" : 3 })
+
+
+        # Fill data
+        self.ok("create /newton/prep float32_8")
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-timestamped") as input:
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
+
+        # Extract it
+        self.ok("extract /newton/prep --start '2000-01-01' " +
+                "--end '2012-03-23 10:04:01'")
+        extract_before = self.captured
+
+        def check_path(*components):
+            # Verify the paths look right on disk
+            seek = os.path.join(testdb, "data", *components)
+            for (dirpath, dirnames, filenames) in os.walk(testdb):
+                if "_format" in filenames:
+                    if dirpath == seek:
+                        break
+                    raise AssertionError("data also found at " + dirpath)
+            else:
+                raise AssertionError("data not found at " + seek)
+            # Verify "list" output
+            self.ok("list")
+            self.match("/" + "/".join(components) + " float32_8\n")
+
+        # Lots of renames
+        check_path("newton", "prep")
+
+        self.fail("rename /newton/prep /newton/prep")
+        self.contain("old and new paths are the same")
+        check_path("newton", "prep")
+        self.fail("rename /newton/prep /newton")
+        self.contain("path must contain at least one folder")
+        self.fail("rename /newton/prep /newton/prep/")
+        self.contain("invalid path")
+        self.ok("rename /newton/prep /newton/foo/1")
+        check_path("newton", "foo", "1")
+        self.ok("rename /newton/foo/1 /newton/foo")
+        check_path("newton", "foo")
+        self.ok("rename /newton/foo /totally/different/thing")
+        check_path("totally", "different", "thing")
+        self.ok("rename /totally/different/thing /totally/something")
+        check_path("totally", "something")
+        self.ok("rename /totally/something /totally/something/cool")
+        check_path("totally", "something", "cool")
+        self.ok("rename /totally/something/cool /foo/bar")
+        check_path("foo", "bar")
+        self.ok("create /xxx/yyy/zzz float32_8")
+        self.fail("rename /foo/bar /xxx/yyy")
+        self.contain("subdirs of this path already exist")
+        self.fail("rename /foo/bar /xxx/yyy/zzz")
+        self.contain("stream already exists at this path")
+        self.fail("rename /foo/bar /xxx/yyy/zzz/www")
+        self.contain("path is subdir of existing node")
+        self.ok("rename /foo/bar /xxx/yyy/mmm")
+        self.ok("destroy -R /xxx/yyy/zzz")
+        check_path("xxx", "yyy", "mmm")
+
+        # Extract it at the final path
+        self.ok("extract /xxx/yyy/mmm --start '2000-01-01' " +
+                "--end '2012-03-23 10:04:01'")
+        eq_(self.captured, extract_before)
+
+        self.ok("destroy -R /xxx/yyy/mmm")
+
+        # Make sure temporary rename dirs weren't left around
+        for (dirpath, dirnames, filenames) in os.walk(testdb):
+            if "rename-" in dirpath:
+                raise AssertionError("temporary directories not cleaned up")
+            if "totally" in dirpath or "newton" in dirpath:
+                raise AssertionError("old directories not cleaned up")
+
+        server_stop()
+        server_start()

tests/test_interval.py

@@ -8,8 +8,11 @@ from nose.tools import *
 from nose.tools import assert_raises
 import itertools
 
-from nilmdb.server.interval import (Interval, DBInterval,
+from nilmdb.utils.interval import IntervalError
-                                    IntervalSet, IntervalError)
+from nilmdb.server.interval import Interval, DBInterval, IntervalSet
+
+# so we can test them separately
+from nilmdb.utils.interval import Interval as UtilsInterval
 
 from testutil.helpers import *
 import unittest
@@ -47,15 +50,24 @@ def makeset(string):
     return iset
 
 class TestInterval:
+    def test_client_interval(self):
+        # Run interval tests against the Python version of Interval.
+        global Interval
+        NilmdbInterval = Interval
+        Interval = UtilsInterval
+        self.test_interval()
+        self.test_interval_intersect()
+        Interval = NilmdbInterval
+
     def test_interval(self):
         # Test Interval class
         os.environ['TZ'] = "America/New_York"
         datetime_tz._localtz = None
-        (d1, d2, d3) = [ datetime_tz.datetime_tz.smartparse(x).totimestamp()
+        (d1, d2, d3) = [ nilmdb.utils.time.parse_time(x)
                          for x in [ "03/24/2012", "03/25/2012", "03/26/2012" ] ]
 
         # basic construction
-        i = Interval(d1, d1)
+        i = Interval(d1, d2)
         i = Interval(d1, d3)
         eq_(i.start, d1)
         eq_(i.end, d3)
@@ -77,8 +89,8 @@ class TestInterval:
         assert(Interval(d1, d3) > Interval(d1, d2))
         assert(Interval(d1, d2) < Interval(d2, d3))
         assert(Interval(d1, d3) < Interval(d2, d3))
-        assert(Interval(d2, d2) > Interval(d1, d3))
+        assert(Interval(d2, d2+1) > Interval(d1, d3))
-        assert(Interval(d3, d3) == Interval(d3, d3))
+        assert(Interval(d3, d3+1) == Interval(d3, d3+1))
         #with assert_raises(TypeError): # was AttributeError, that's wrong
         #    x = (i == 123)
 
@@ -87,16 +99,16 @@ class TestInterval:
         with assert_raises(IntervalError):
             x = Interval(d2, d3).subset(d1, d2)
 
-        # big integers and floats
+        # big integers, negative integers
-        x = Interval(5000111222, 6000111222)
+        x = Interval(5000111222000000, 6000111222000000)
-        eq_(str(x), "[5000111222.0 -> 6000111222.0)")
+        eq_(str(x), "[5000111222000000 -> 6000111222000000)")
-        x = Interval(123.45, 234.56)
+        x = Interval(-5000111222000000, -4000111222000000)
-        eq_(str(x), "[123.45 -> 234.56)")
+        eq_(str(x), "[-5000111222000000 -> -4000111222000000)")
 
         # misc
         i = Interval(d1, d2)
         eq_(repr(i), repr(eval(repr(i))))
-        eq_(str(i), "[1332561600.0 -> 1332648000.0)")
+        eq_(str(i), "[1332561600000000 -> 1332648000000000)")
 
     def test_interval_intersect(self):
         # Test Interval intersections
@@ -192,7 +204,8 @@ class TestInterval:
 
         # misc
         eq_(repr(iset), repr(eval(repr(iset))))
-        eq_(str(iset), "[[100.0 -> 200.0), [200.0 -> 300.0)]")
+        eq_(str(iset),
+            "[[100 -> 200), [200 -> 300)]")
 
     def test_intervalset_geniset(self):
         # Test basic iset construction
@@ -207,64 +220,90 @@ class TestInterval:
             makeset("  [-|-----|"))
 
 
-    def test_intervalset_intersect(self):
+    def test_intervalset_intersect_difference(self):
         # Test intersection (&)
         with assert_raises(TypeError): # was AttributeError
             x = makeset("[--)") & 1234
 
-        # Intersection with interval
+        def do_test(a, b, c, d):
-        eq_(makeset("[---|---)[)") &
+            # a & b == c
-            list(makeset("  [------) "))[0],
+            ab = IntervalSet()
-            makeset("  [-----)  "))
+            for x in b:
+                for i in (a & x):
+                    ab += i
+            eq_(ab,c)
 
-        # Intersection with sets
+            # a \ b == d
-        eq_(makeset("[---------)") &
+            eq_(IntervalSet(nilmdb.utils.interval.set_difference(a,b)), d)
-            makeset(" [---)     "),
-            makeset(" [---)     "))
 
-        eq_(makeset(" [---)     ") &
+        # Intersection with intervals
-            makeset("[---------)"),
+        do_test(makeset("[---|---)[)"),
-            makeset(" [---)     "))
+                makeset("  [------) "),
+                makeset("  [-----)  "), # intersection
+                makeset("[-)      [)")) # difference
 
-        eq_(makeset("    [-----)") &
+        do_test(makeset("[---------)"),
-            makeset(" [-----)   "),
+                makeset(" [---)     "),
-            makeset("    [--)   "))
+                makeset(" [---)     "), # intersection
+                makeset("[)   [----)")) # difference
 
-        eq_(makeset(" [--)  [--)") &
+        do_test(makeset(" [---)     "),
-            makeset("  [------) "),
+                makeset("[---------)"),
-            makeset("  [-)  [-) "))
+                makeset(" [---)     "), # intersection
+                makeset("           ")) # difference
 
-        eq_(makeset("      [---)") &
+        do_test(makeset("    [-----)"),
-            makeset(" [--)      "),
+                makeset(" [-----)   "),
-            makeset("           "))
+                makeset("    [--)   "), # intersection
+                makeset("       [--)")) # difference
 
-        eq_(makeset("    [-|---)") &
+        do_test(makeset(" [--)  [--)"),
-            makeset(" [-----|-) "),
+                makeset("  [------) "),
-            makeset("    [----) "))
+                makeset("  [-)  [-) "), # intersection
+                makeset(" [)      [)")) # difference
 
-        eq_(makeset("    [-|-)  ") &
+        do_test(makeset("      [---)"),
-            makeset(" [-|--|--) "),
+                makeset(" [--)      "),
-            makeset("    [---)  "))
+                makeset("           "), # intersection
+                makeset("      [---)")) # difference
+
+        do_test(makeset("    [-|---)"),
+                makeset(" [-----|-) "),
+                makeset("    [----) "), # intersection
+                makeset("         [)")) # difference
+
+        do_test(makeset("    [-|-)  "),
+                makeset(" [-|--|--) "),
+                makeset("    [---)  "), # intersection
+                makeset("           ")) # difference
+
+        do_test(makeset("[-)[-)[-)[)"),
+                makeset(" [)  [|)[) "),
+                makeset(" [)   [)   "), # intersection
+                makeset("[) [-) [)[)")) # difference
 
         # Border cases -- will give different results if intervals are
-        # half open or fully closed.  Right now, they are half open,
+        # half open or fully closed.  In nilmdb, they are half open.
-        # although that's a little messy since the database intervals
+        do_test(makeset("      [---)"),
-        # often contain a data point at the endpoint.
-        half_open = True
-        if half_open:
-            eq_(makeset("      [---)") &
                 makeset(" [----)    "),
-                makeset("           "))
+                makeset("           "), # intersection
-            eq_(makeset(" [----)[--)") &
+                makeset("      [---)")) # difference
+
+        do_test(makeset(" [----)[--)"),
                 makeset("[-) [--) [)"),
-                makeset(" [) [-)  [)"))
+                makeset(" [) [-)  [)"), # intersection
-        else:
+                makeset("  [-)  [-) ")) # difference
-            eq_(makeset("      [---)") &
+
-                makeset(" [----)    "),
+        # Set difference with bounds
-                makeset("      .    "))
+        a = makeset(" [----)[--)")
-            eq_(makeset(" [----)[--)") &
+        b = makeset("[-) [--) [)")
-                makeset("[-) [--) [)"),
+        c = makeset("[----)     ")
-                makeset(" [) [-). [)"))
+        d = makeset("  [-)      ")
+        eq_(nilmdb.utils.interval.set_difference(
+            a.intersection(list(c)[0]), b.intersection(list(c)[0])), d)
+
+        # Empty second set
+        eq_(nilmdb.utils.interval.set_difference(a, IntervalSet()), a)
 
 class TestIntervalDB:
     def test_dbinterval(self):
@@ -293,7 +332,7 @@ class TestIntervalDB:
         # actual start, end can be a subset
         a = DBInterval(150, 200, 100, 200, 10000, 20000)
         b = DBInterval(100, 150, 100, 200, 10000, 20000)
-        c = DBInterval(150, 150, 100, 200, 10000, 20000)
+        c = DBInterval(150, 160, 100, 200, 10000, 20000)
 
         # Make a set of DBIntervals
         iseta = IntervalSet([a, b])
@@ -346,14 +385,13 @@ class TestIntervalSpeed:
     def test_interval_speed(self):
         import yappi
         import time
-        import testutil.aplotter as aplotter
         import random
         import math
 
         print
         yappi.start()
         speeds = {}
-        limit = 10 # was 20
+        limit = 22 # was 20
         for j in [ 2**x for x in range(5,limit) ]:
             start = time.time()
             iset = IntervalSet()
@@ -367,7 +405,5 @@ class TestIntervalSpeed:
                    speed/j,
                    speed / (j*math.log(j))) # should be constant
             speeds[j] = speed
-        aplotter.plot(speeds.keys(), speeds.values(), plot_slope=True)
         yappi.stop()
         yappi.print_stats(sort_type=yappi.SORTTYPE_TTOT, limit=10)
-

tests/test_iteratorizer.py

@@ -1,61 +0,0 @@
-import nilmdb
-from nilmdb.utils.printf import *
-
-import nose
-from nose.tools import *
-from nose.tools import assert_raises
-import threading
-import time
-
-from testutil.helpers import *
-
-def func_with_callback(a, b, callback):
-    callback(a)
-    callback(b)
-    callback(a+b)
-    return "return value"
-
-class TestIteratorizer(object):
-    def test(self):
-
-        # First try it with a normal callback
-        self.result = ""
-        def cb(x):
-            self.result += str(x)
-        func_with_callback(1, 2, cb)
-        eq_(self.result, "123")
-
-        # Now make it an iterator
-        result = ""
-        f = lambda x: func_with_callback(1, 2, x)
-        with nilmdb.utils.Iteratorizer(f) as it:
-            for i in it:
-                result += str(i)
-        eq_(result, "123")
-        eq_(it.retval, "return value")
-
-        # Make sure things work when an exception occurs
-        result = ""
-        with nilmdb.utils.Iteratorizer(
-            lambda x: func_with_callback(1, "a", x)) as it:
-            with assert_raises(TypeError) as e:
-                for i in it:
-                    result += str(i)
-        eq_(result, "1a")
-
-        # Now try to trigger the case where we stop iterating
-        # mid-generator, and expect the iteratorizer to clean up after
-        # itself.  This doesn't have a particular result in the test,
-        # but gains coverage.
-        def foo():
-            with nilmdb.utils.Iteratorizer(f) as it:
-                it.next()
-        foo()
-        eq_(it.retval, None)
-
-        # Do the same thing when the curl hack is applied
-        def foo():
-            with nilmdb.utils.Iteratorizer(f, curl_hack = True) as it:
-                it.next()
-        foo()
-        eq_(it.retval, None)

tests/test_layout.py

@@ -1,266 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import nilmdb
-
-from nilmdb.utils.printf import *
-
-from nose.tools import *
-from nose.tools import assert_raises
-import distutils.version
-import itertools
-import os
-import shutil
-import sys
-import cherrypy
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
-import cStringIO
-import random
-import unittest
-
-from testutil.helpers import *
-
-from nilmdb.server.layout import *
-
-class TestLayouts(object):
-    # Some nilmdb.layout tests.  Not complete, just fills in missing
-    # coverage.
-    def test_layouts(self):
-        x = nilmdb.server.layout.get_named("PrepData")
-        y = nilmdb.server.layout.get_named("float32_8")
-        eq_(x.count, y.count)
-        eq_(x.datatype, y.datatype)
-        y = nilmdb.server.layout.get_named("float32_7")
-        ne_(x.count, y.count)
-        eq_(x.datatype, y.datatype)
-
-    def test_parsing(self):
-        self.real_t_parsing("PrepData", "RawData", "RawNotchedData")
-        self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
-    def real_t_parsing(self, name_prep, name_raw, name_rawnotch):
-        # invalid layouts
-        with assert_raises(TypeError) as e:
-            parser = Parser("NoSuchLayout")
-        with assert_raises(TypeError) as e:
-            parser = Parser("float32")
-
-        # too little data
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5\n")
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("error", str(e.exception))
-
-        # too much data
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n")
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("error", str(e.exception))
-
-        # just right
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n")
-        parser.parse(data)
-        eq_(parser.min_timestamp, 1234567890.0)
-        eq_(parser.max_timestamp, 1234567890.1)
-        eq_(parser.data, [[1234567890.0,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8],
-                          [1234567890.1,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]])
-
-        # try RawData too, with clamping
-        parser = Parser(name_raw)
-        data = ( "1234567890.000000 1 2 3 4 5 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        parser.parse(data)
-        eq_(parser.data, [[1234567890.0,1,2,3,4,5,6],
-                          [1234567890.1,1,2,3,4,5,6]])
-
-        # pass an instantiated class
-        parser = Parser(get_named(name_rawnotch))
-        data = ( "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
-                 "1234567890.100000 1 2 3 4 5 6 7 8 9\n" )
-        parser.parse(data)
-
-        # non-monotonic
-        parser = Parser(name_raw)
-        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.099999 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("not monotonically increasing", str(e.exception))
-
-        parser = Parser(name_raw)
-        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("not monotonically increasing", str(e.exception))
-
-        parser = Parser(name_raw)
-        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.100001 1 2 3 4 5 6\n" )
-        parser.parse(data)
-
-        # RawData with values out of bounds
-        parser = Parser(name_raw)
-        data = ( "1234567890.000000 1 2 3 4 500000 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("value out of range", str(e.exception))
-
-        # Empty data should work but is useless
-        parser = Parser(name_raw)
-        data = ""
-        parser.parse(data)
-        assert(parser.min_timestamp is None)
-        assert(parser.max_timestamp is None)
-
-    def test_formatting(self):
-        self.real_t_formatting("PrepData", "RawData", "RawNotchedData")
-        self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
-    def real_t_formatting(self, name_prep, name_raw, name_rawnotch):
-        # invalid layout
-        with assert_raises(TypeError) as e:
-            formatter = Formatter("NoSuchLayout")
-
-        # too little data
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5 ],
-                 [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5 ] ]
-        with assert_raises(FormatterError) as e:
-            formatter.format(data)
-        in_("error", str(e.exception))
-
-        # too much data
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
-        with assert_raises(FormatterError) as e:
-            formatter.format(data)
-        in_("error", str(e.exception))
-
-        # just right
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ],
-                 [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1.100000 2.200000 3.300000 4.400000 " +
-            "5.500000 6.600000 7.700000 8.800000\n" +
-            "1234567890.100000 1.100000 2.200000 3.300000 4.400000 " +
-            "5.500000 6.600000 7.700000 8.800000\n")
-
-        # try RawData too
-        formatter = Formatter(name_raw)
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1 2 3 4 5 6\n" +
-            "1234567890.100000 1 2 3 4 5 6\n")
-
-        # pass an instantiated class
-        formatter = Formatter(get_named(name_rawnotch))
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
-            "1234567890.100000 1 2 3 4 5 6 7 8 9\n")
-
-        # Empty data should work but is useless
-        formatter = Formatter(name_raw)
-        data = []
-        text = formatter.format(data)
-        eq_(text, "")
-
-    def test_roundtrip(self):
-        self.real_t_roundtrip("PrepData", "RawData", "RawNotchedData")
-        self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
-    def real_t_roundtrip(self, name_prep, name_raw, name_rawnotch):
-        # Verify that textual data passed into the Parser, and then
-        # back through the Formatter, then back into the Parser,
-        # gives identical parsed representations
-        random.seed(12345)
-
-        def do_roundtrip(layout, datagen):
-            for i in range(100):
-                rows = random.randint(1,100)
-                data = ""
-                ts = 1234567890
-                for r in range(rows):
-                    ts += random.uniform(0,1)
-                    row = sprintf("%f", ts) + " "
-                    row += " ".join(datagen())
-                    row += "\n"
-                    data += row
-                parser1 = Parser(layout)
-                formatter = Formatter(layout)
-                parser2 = Parser(layout)
-                parser1.parse(data)
-                parser2.parse(formatter.format(parser1.data))
-                eq_(parser1.data, parser2.data)
-
-        def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
-                     for x in range(8) ]
-        do_roundtrip(name_prep, datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(6) ]
-        do_roundtrip(name_raw, datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(9) ]
-        do_roundtrip(name_rawnotch, datagen)
-
-class TestLayoutSpeed:
-    @unittest.skip("this is slow")
-    def test_layout_speed(self):
-        import time
-
-        random.seed(54321)
-
-        def do_speedtest(layout, datagen, rows = 5000, times = 100):
-            # Build data once
-            data = ""
-            ts = 1234567890
-            for r in range(rows):
-                ts += random.uniform(0,1)
-                row = sprintf("%f", ts) + " "
-                row += " ".join(datagen())
-                row += "\n"
-                data += row
-
-            # Do lots of roundtrips
-            start = time.time()
-            for i in range(times):
-                parser = Parser(layout)
-                formatter = Formatter(layout)
-                parser.parse(data)
-                data = formatter.format(parser.data)
-            elapsed = time.time() - start
-            printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
-                   layout,
-                   elapsed * 1e3,
-                   (elapsed * 1e6) / (rows * times),
-                   (rows * times) / elapsed)
-
-        print ""
-        def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
-                     for x in range(10) ]
-        do_speedtest("float32_10", datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(10) ]
-        do_speedtest("uint16_10", datagen)

tests/test_mustclose.py

@@ -34,6 +34,10 @@ class Bar:
     def __del__(self):
         fprintf(err, "Deleting\n")
 
+    @classmethod
+    def baz(self):
+        fprintf(err, "Baz\n")
+
     def close(self):
         fprintf(err, "Closing\n")
 

tests/test_nilmdb.py

@@ -1,4 +1,4 @@
-import nilmdb
+import nilmdb.server
 
 from nose.tools import *
 from nose.tools import assert_raises
@@ -6,15 +6,15 @@ import distutils.version
 import simplejson as json
 import itertools
 import os
-import shutil
 import sys
-import cherrypy
 import threading
 import urllib2
 from urllib2 import urlopen, HTTPError
-import Queue
 import cStringIO
 import time
+import requests
+
+from nilmdb.utils import serializer_proxy
 
 testdb = "tests/testdb"
 
@@ -28,12 +28,9 @@ class Test00Nilmdb(object):  # named 00 so it runs first
     def test_NilmDB(self):
         recursive_unlink(testdb)
 
-        with assert_raises(IOError):
+        db = nilmdb.server.NilmDB(testdb)
-            nilmdb.NilmDB("/nonexistant-db/foo")
-
-        db = nilmdb.NilmDB(testdb)
         db.close()
-        db = nilmdb.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         db.close()
 
         # test timer, just to get coverage
@@ -46,29 +43,29 @@ class Test00Nilmdb(object):  # named 00 so it runs first
         in_("test: ", capture.getvalue())
 
     def test_stream(self):
-        db = nilmdb.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         eq_(db.stream_list(), [])
 
         # Bad path
         with assert_raises(ValueError):
-            db.stream_create("foo/bar/baz", "PrepData")
+            db.stream_create("foo/bar/baz", "float32_8")
         with assert_raises(ValueError):
-            db.stream_create("/foo", "PrepData")
+            db.stream_create("/foo", "float32_8")
         # Bad layout type
         with assert_raises(ValueError):
             db.stream_create("/newton/prep", "NoSuchLayout")
-        db.stream_create("/newton/prep", "PrepData")
+        db.stream_create("/newton/prep", "float32_8")
-        db.stream_create("/newton/raw", "RawData")
+        db.stream_create("/newton/raw", "uint16_6")
-        db.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
+        db.stream_create("/newton/zzz/rawnotch", "uint16_9")
 
         # Verify we got 3 streams
-        eq_(db.stream_list(), [ ["/newton/prep", "PrepData"],
+        eq_(db.stream_list(), [ ["/newton/prep", "float32_8"],
-                                ["/newton/raw", "RawData"],
+                                ["/newton/raw", "uint16_6"],
-                                ["/newton/zzz/rawnotch", "RawNotchedData"]
+                                ["/newton/zzz/rawnotch", "uint16_9"]
                                 ])
         # Match just one type or one path
-        eq_(db.stream_list(layout="RawData"), [ ["/newton/raw", "RawData"] ])
+        eq_(db.stream_list(layout="uint16_6"), [ ["/newton/raw", "uint16_6"] ])
-        eq_(db.stream_list(path="/newton/raw"), [ ["/newton/raw", "RawData"] ])
+        eq_(db.stream_list(path="/newton/raw"), [ ["/newton/raw", "uint16_6"] ])
 
         # Verify that columns were made right (pytables specific)
         if "h5file" in db.data.__dict__:
@@ -93,19 +90,34 @@ class Test00Nilmdb(object):  # named 00 so it runs first
         eq_(db.stream_get_metadata("/newton/prep"), meta1)
         eq_(db.stream_get_metadata("/newton/raw"), meta1)
 
+        # fill in some misc. test coverage
+        with assert_raises(nilmdb.server.NilmDBError):
+            db.stream_remove("/newton/prep", 0, 0)
+        with assert_raises(nilmdb.server.NilmDBError):
+            db.stream_remove("/newton/prep", 1, 0)
+        db.stream_remove("/newton/prep", 0, 1)
+
+        with assert_raises(nilmdb.server.NilmDBError):
+            db.stream_extract("/newton/prep", count = True, binary = True)
+
         db.close()
 
 class TestBlockingServer(object):
     def setUp(self):
-        self.db = nilmdb.NilmDB(testdb, sync=False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
 
     def tearDown(self):
         self.db.close()
 
     def test_blocking_server(self):
+        # Server should fail if the database doesn't have a "_thread_safe"
+        # property.
+        with assert_raises(KeyError):
+            nilmdb.server.Server(object())
+
         # Start web app on a custom port
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
+        self.server = nilmdb.server.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = True)
+                                           port = 32180, stoppable = True)
 
         # Run it
         event = threading.Event()
@@ -117,13 +129,13 @@ class TestBlockingServer(object):
             raise AssertionError("server didn't start in 10 seconds")
 
         # Send request to exit.
-        req = urlopen("http://127.0.0.1:12380/exit/", timeout = 1)
+        req = urlopen("http://127.0.0.1:32180/exit/", timeout = 1)
 
         # Wait for it
         thread.join()
 
 def geturl(path):
-    req = urlopen("http://127.0.0.1:12380" + path, timeout = 10)
+    req = urlopen("http://127.0.0.1:32180" + path, timeout = 10)
     return req.read()
 
 def getjson(path):
@@ -133,9 +145,9 @@ class TestServer(object):
 
     def setUp(self):
         # Start web app on a custom port
-        self.db = nilmdb.NilmDB(testdb, sync=False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
+        self.server = nilmdb.server.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = False)
+                                           port = 32180, stoppable = False)
         self.server.start(blocking = False)
 
     def tearDown(self):
@@ -151,21 +163,21 @@ class TestServer(object):
             eq_(e.exception.code, 404)
 
         # Check version
-        eq_(distutils.version.StrictVersion(getjson("/version")),
+        eq_(distutils.version.LooseVersion(getjson("/version")),
-            distutils.version.StrictVersion(self.server.version))
+            distutils.version.LooseVersion(nilmdb.__version__))
 
     def test_stream_list(self):
         # Known streams that got populated by an earlier test (test_nilmdb)
         streams = getjson("/stream/list")
 
         eq_(streams, [
-            ['/newton/prep', 'PrepData'],
+            ['/newton/prep', 'float32_8'],
-            ['/newton/raw', 'RawData'],
+            ['/newton/raw', 'uint16_6'],
-            ['/newton/zzz/rawnotch', 'RawNotchedData'],
+            ['/newton/zzz/rawnotch', 'uint16_9'],
             ])
 
-        streams = getjson("/stream/list?layout=RawData")
+        streams = getjson("/stream/list?layout=uint16_6")
-        eq_(streams, [['/newton/raw', 'RawData']])
+        eq_(streams, [['/newton/raw', 'uint16_6']])
 
         streams = getjson("/stream/list?layout=NoSuchLayout")
         eq_(streams, [])
@@ -195,11 +207,50 @@ class TestServer(object):
                        "&key=foo")
         eq_(data, {'foo': None})
 
+    def test_cors_headers(self):
+        # Test that CORS headers are being set correctly
 
-    def test_insert(self):
+        # Normal GET should send simple response
-        # GET instead of POST (no body)
+        url = "http://127.0.0.1:32180/stream/list"
-        # (actual POST test is done by client code)
+        r = requests.get(url, headers = { "Origin": "http://google.com/" })
-        with assert_raises(HTTPError) as e:
+        eq_(r.status_code, 200)
-            getjson("/stream/insert?path=/newton/prep&start=0&end=0")
+        if "access-control-allow-origin" not in r.headers:
-        eq_(e.exception.code, 400)
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-origin"], "http://google.com/")
 
+        # OPTIONS without CORS preflight headers should result in 405
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            })
+        eq_(r.status_code, 405)
+
+        # OPTIONS with preflight headers should give preflight response
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            "Access-Control-Request-Method": "POST",
+            "Access-Control-Request-Headers": "X-Custom",
+            })
+        eq_(r.status_code, 200)
+        if "access-control-allow-origin" not in r.headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-methods"], "GET, HEAD")
+        eq_(r.headers["access-control-allow-headers"], "X-Custom")
+
+    def test_post_bodies(self):
+        # Test JSON post bodies
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '{"hello": 1}')
+        eq_(r.status_code, 404) # wrong parameters
+
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '["hello"]')
+        eq_(r.status_code, 415) # not a dict
+
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '[hello]')
+        eq_(r.status_code, 400) # badly formatted JSON

tests/test_numpyclient.py

@@ -0,0 +1,333 @@
+# -*- coding: utf-8 -*-
+
+import nilmdb.server
+import nilmdb.client
+import nilmdb.client.numpyclient
+
+from nilmdb.utils.printf import *
+from nilmdb.utils import timestamper
+from nilmdb.client import ClientError, ServerError
+from nilmdb.utils import datetime_tz
+
+from nose.plugins.skip import SkipTest
+from nose.tools import *
+from nose.tools import assert_raises
+import itertools
+import distutils.version
+
+from testutil.helpers import *
+
+import numpy as np
+
+testdb = "tests/numpyclient-testdb"
+testurl = "http://localhost:32180/"
+
+def setup_module():
+    global test_server, test_db
+    # Clear out DB
+    recursive_unlink(testdb)
+
+    # Start web app on a custom port
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(testdb)
+    test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
+                                       port = 32180, stoppable = False,
+                                       fast_shutdown = True,
+                                       force_traceback = True)
+    test_server.start(blocking = False)
+
+def teardown_module():
+    global test_server, test_db
+    # Close web app
+    test_server.stop()
+    test_db.close()
+
+class TestNumpyClient(object):
+
+    def test_numpyclient_01_basic(self):
+        # Test basic connection
+        client = nilmdb.client.numpyclient.NumpyClient(url = testurl)
+        version = client.version()
+        eq_(distutils.version.LooseVersion(version),
+            distutils.version.LooseVersion(test_server.version))
+
+        # Verify subclassing
+        assert(isinstance(client, nilmdb.client.Client))
+
+        # Layouts
+        for layout in "int8_t", "something_8", "integer_1":
+            with assert_raises(ValueError):
+                for x in client.stream_extract_numpy("/foo", layout=layout):
+                    pass
+        for layout in "int8_1", "uint8_30", "int16_20", "float64_100":
+            with assert_raises(ClientError) as e:
+                for x in client.stream_extract_numpy("/foo", layout=layout):
+                    pass
+            in_("No such stream", str(e.exception))
+
+        with assert_raises(ClientError) as e:
+            for x in client.stream_extract_numpy("/foo"):
+                pass
+        in_("can't get layout for path", str(e.exception))
+
+        client.close()
+
+    def test_numpyclient_02_extract(self):
+        client = nilmdb.client.numpyclient.NumpyClient(url = testurl)
+
+        # Insert some data as text
+        client.stream_create("/newton/prep", "float32_8")
+        testfile = "tests/data/prep-20120323T1000"
+        start = nilmdb.utils.time.parse_time("20120323T1000")
+        rate = 120
+        data = timestamper.TimestamperRate(testfile, start, rate)
+        result = client.stream_insert("/newton/prep", data,
+                                      start, start + 119999777)
+
+        # Extract Numpy arrays
+        array = None
+        pieces = 0
+        for chunk in client.stream_extract_numpy("/newton/prep", maxrows=1000):
+            pieces += 1
+            if array is not None:
+                array = np.vstack((array, chunk))
+            else:
+                array = chunk
+        eq_(array.shape, (14400, 9))
+        eq_(pieces, 15)
+
+        # Try structured
+        s = list(client.stream_extract_numpy("/newton/prep", structured = True))
+        assert(np.array_equal(np.c_[s[0]['timestamp'], s[0]['data']], array))
+
+        # Compare.  Will be close but not exact because the conversion
+        # to and from ASCII was lossy.
+        data = timestamper.TimestamperRate(testfile, start, rate)
+        actual = np.fromstring(" ".join(data), sep=' ').reshape(14400, 9)
+        assert(np.allclose(array, actual))
+
+        client.close()
+
+    def test_numpyclient_03_insert(self):
+        client = nilmdb.client.numpyclient.NumpyClient(url = testurl)
+
+        # Limit _max_data just to get better coverage
+        old_max_data = nilmdb.client.numpyclient.StreamInserterNumpy._max_data
+        nilmdb.client.numpyclient.StreamInserterNumpy._max_data = 100000
+
+        client.stream_create("/test/1", "uint16_1")
+        client.stream_insert_numpy("/test/1",
+                                   np.array([[0, 1],
+                                             [1, 2],
+                                             [2, 3],
+                                             [3, 4]]))
+
+        # Wrong number of dimensions
+        with assert_raises(ValueError) as e:
+            client.stream_insert_numpy("/test/1",
+                                       np.array([[[0, 1],
+                                                  [1, 2]],
+                                                 [[3, 4],
+                                                  [4, 5]]]))
+        in_("wrong number of dimensions", str(e.exception))
+
+        # Unstructured
+        client.stream_create("/test/2", "float32_8")
+        client.stream_insert_numpy(
+            "/test/2",
+            client.stream_extract_numpy(
+                "/newton/prep", structured = False, maxrows = 1000))
+
+        # Structured, and specifying layout
+        client.stream_create("/test/3", "float32_8")
+        client.stream_insert_numpy(
+            path = "/test/3", layout = "float32_8",
+            data = client.stream_extract_numpy(
+                "/newton/prep", structured = True, maxrows = 1000))
+
+        # Structured, specifying wrong layout
+        client.stream_create("/test/4", "float32_8")
+        with assert_raises(ValueError) as e:
+            client.stream_insert_numpy(
+                "/test/4", layout = "uint16_1",
+                data = client.stream_extract_numpy(
+                    "/newton/prep", structured = True, maxrows = 1000))
+        in_("wrong dtype", str(e.exception))
+
+        # Unstructured, and specifying wrong layout
+        client.stream_create("/test/5", "float32_8")
+        with assert_raises(ClientError) as e:
+            client.stream_insert_numpy(
+                "/test/5", layout = "uint16_8",
+                data = client.stream_extract_numpy(
+                    "/newton/prep", structured = False, maxrows = 1000))
+        # timestamps will be screwy here, because data will be parsed wrong
+        in_("error parsing input data", str(e.exception))
+
+        # Make sure the /newton/prep copies are identical
+        a = np.vstack(client.stream_extract_numpy("/newton/prep"))
+        b = np.vstack(client.stream_extract_numpy("/test/2"))
+        c = np.vstack(client.stream_extract_numpy("/test/3"))
+        assert(np.array_equal(a,b))
+        assert(np.array_equal(a,c))
+
+        nilmdb.client.numpyclient.StreamInserterNumpy._max_data = old_max_data
+        client.close()
+
+    def test_numpyclient_04_context(self):
+        # Like test_client_context, but with Numpy data
+        client = nilmdb.client.numpyclient.NumpyClient(testurl)
+
+        client.stream_create("/context/test", "uint16_1")
+        with client.stream_insert_numpy_context("/context/test") as ctx:
+            # override _max_rows to trigger frequent server updates
+            ctx._max_rows = 2
+            ctx.insert([[1000, 1]])
+            ctx.insert([[1010, 1], [1020, 1], [1030, 1]])
+            ctx.insert([[1040, 1], [1050, 1]])
+            ctx.finalize()
+            ctx.insert([[1070, 1]])
+            ctx.update_end(1080)
+            ctx.finalize()
+            ctx.update_start(1090)
+            ctx.insert([[1100, 1]])
+            ctx.insert([[1110, 1]])
+            ctx.send()
+            ctx.insert([[1120, 1], [1130, 1], [1140, 1]])
+            ctx.update_end(1160)
+            ctx.insert([[1150, 1]])
+            ctx.update_end(1170)
+            ctx.insert([[1160, 1]])
+            ctx.update_end(1180)
+            ctx.insert([[1170, 123456789.0]])
+            ctx.finalize()
+            ctx.insert(np.zeros((0,2)))
+
+        with assert_raises(ClientError):
+            with client.stream_insert_numpy_context("/context/test",
+                                                    1000, 2000) as ctx:
+                ctx.insert([[1180, 1]])
+
+        with assert_raises(ClientError):
+            with client.stream_insert_numpy_context("/context/test",
+                                                    2000, 3000) as ctx:
+                ctx._max_rows = 2
+                ctx.insert([[3180, 1]])
+                ctx.insert([[3181, 1]])
+
+        with client.stream_insert_numpy_context("/context/test",
+                                                2000, 3000) as ctx:
+            # make sure our override wasn't permanent
+            ne_(ctx._max_rows, 2)
+            ctx.insert([[2250, 1]])
+            ctx.finalize()
+
+        with assert_raises(ClientError):
+            with client.stream_insert_numpy_context("/context/test",
+                                                    3000, 4000) as ctx:
+                ctx.insert([[3010, 1]])
+                ctx.insert([[3020, 2]])
+                ctx.insert([[3030, 3]])
+                ctx.insert([[3040, 4]])
+                ctx.insert([[3040, 4]]) # non-monotonic after a few lines
+                ctx.finalize()
+
+        eq_(list(client.stream_intervals("/context/test")),
+            [ [ 1000, 1051 ],
+              [ 1070, 1080 ],
+              [ 1090, 1180 ],
+              [ 2000, 3000 ] ])
+
+        client.stream_remove("/context/test")
+        client.stream_destroy("/context/test")
+        client.close()
+
+    def test_numpyclient_05_emptyintervals(self):
+        # Like test_client_emptyintervals, with insert_numpy_context
+        client = nilmdb.client.numpyclient.NumpyClient(testurl)
+        client.stream_create("/empty/test", "uint16_1")
+        def info():
+            result = []
+            for interval in list(client.stream_intervals("/empty/test")):
+                result.append((client.stream_count("/empty/test", *interval),
+                               interval))
+            return result
+        eq_(info(), [])
+
+        # Insert a region with just a few points
+        with client.stream_insert_numpy_context("/empty/test") as ctx:
+            ctx.update_start(100)
+            ctx.insert([[140, 1]])
+            ctx.insert([[150, 1]])
+            ctx.insert([[160, 1]])
+            ctx.update_end(200)
+            ctx.finalize()
+        eq_(info(), [(3, [100, 200])])
+
+        # Delete chunk, which will leave one data point and two intervals
+        client.stream_remove("/empty/test", 145, 175)
+        eq_(info(), [(1, [100, 145]),
+                     (0, [175, 200])])
+
+        # Try also creating a completely empty interval from scratch,
+        # in a few different ways.
+        client.stream_insert("/empty/test", "", 300, 350)
+        client.stream_insert("/empty/test", [], 400, 450)
+        with client.stream_insert_numpy_context("/empty/test", 500, 550):
+            pass
+
+        # If enough timestamps aren't provided, empty streams won't be created.
+        client.stream_insert("/empty/test", [])
+        with client.stream_insert_numpy_context("/empty/test"):
+            pass
+        client.stream_insert("/empty/test", [], start = 600)
+        with client.stream_insert_numpy_context("/empty/test", start = 700):
+            pass
+        client.stream_insert("/empty/test", [], end = 850)
+        with client.stream_insert_numpy_context("/empty/test", end = 950):
+            pass
+
+        # Try various things that might cause problems
+        with client.stream_insert_numpy_context("/empty/test", 1000, 1050):
+            ctx.finalize() # inserts [1000, 1050]
+            ctx.finalize() # nothing
+            ctx.finalize() # nothing
+            ctx.insert([[1100, 1]])
+            ctx.finalize() # inserts [1100, 1101]
+            ctx.update_start(1199)
+            ctx.insert([[1200, 1]])
+            ctx.update_end(1250)
+            ctx.finalize() # inserts [1199, 1250]
+            ctx.update_start(1299)
+            ctx.finalize() # nothing
+            ctx.update_end(1350)
+            ctx.finalize() # nothing
+            ctx.update_start(1400)
+            ctx.insert(np.zeros((0,2)))
+            ctx.update_end(1450)
+            ctx.finalize()
+            ctx.update_start(1500)
+            ctx.insert(np.zeros((0,2)))
+            ctx.update_end(1550)
+            ctx.finalize()
+            ctx.insert(np.zeros((0,2)))
+            ctx.insert(np.zeros((0,2)))
+            ctx.insert(np.zeros((0,2)))
+            ctx.finalize()
+
+        # Check everything
+        eq_(info(), [(1, [100, 145]),
+                     (0, [175, 200]),
+                     (0, [300, 350]),
+                     (0, [400, 450]),
+                     (0, [500, 550]),
+                     (0, [1000, 1050]),
+                     (1, [1100, 1101]),
+                     (1, [1199, 1250]),
+                     (0, [1400, 1450]),
+                     (0, [1500, 1550]),
+                     ])
+
+        # Clean up
+        client.stream_remove("/empty/test")
+        client.stream_destroy("/empty/test")
+        client.close()

tests/test_printf.py

@@ -18,7 +18,7 @@ class TestPrintf(object):
             printf("hello, world: %d", 123)
             fprintf(test2, "hello too: %d", 123)
             test3 = sprintf("hello three: %d", 123)
-        except:
+        except Exception:
             sys.stdout = old_stdout
             raise
         sys.stdout = old_stdout

tests/test_serializer.py

@@ -9,16 +9,28 @@ import time
 
 from testutil.helpers import *
 
-#raise nose.exc.SkipTest("Skip these")
-
 class Foo(object):
     val = 0
 
+    def __init__(self, asdf = "asdf"):
+        self.init_thread = threading.current_thread().name
+
+    @classmethod
+    def foo(self):
+        pass
+
     def fail(self):
         raise Exception("you asked me to do this")
 
     def test(self, debug = False):
+        self.tester(debug)
+
+    def t(self):
+        pass
+
+    def tester(self, debug = False):
         # purposely not thread-safe
+        self.test_thread = threading.current_thread().name
         oldval = self.val
         newval = oldval + 1
         time.sleep(0.05)
@@ -46,27 +58,29 @@ class Base(object):
             t.join()
         self.verify_result()
 
+    def verify_result(self):
+        eq_(self.foo.val, 20)
+        eq_(self.foo.init_thread, self.foo.test_thread)
+
 class TestUnserialized(Base):
     def setUp(self):
         self.foo = Foo()
 
     def verify_result(self):
         # This should have failed to increment properly
-        assert(self.foo.val != 20)
+        ne_(self.foo.val, 20)
+        # Init and tests ran in different threads
+        ne_(self.foo.init_thread, self.foo.test_thread)
 
-class TestSerialized(Base):
+class TestSerializer(Base):
     def setUp(self):
-        self.realfoo = Foo()
+        self.foo = nilmdb.utils.serializer_proxy(Foo)("qwer")
-        self.foo = nilmdb.utils.Serializer(self.realfoo)
 
-    def tearDown(self):
+    def test_multi(self):
-        del self.foo
+        sp = nilmdb.utils.serializer_proxy
-
+        sp(Foo("x")).t()
-    def verify_result(self):
+        sp(sp(Foo)("x")).t()
-        # This should have worked
+        sp(sp(Foo))("x").t()
-        eq_(self.realfoo.val, 20)
+        sp(sp(Foo("x"))).t()
-
+        sp(sp(Foo)("x")).t()
-    def test_attribute(self):
+        sp(sp(Foo))("x").t()
-        # Can't wrap attributes yet
-        with assert_raises(TypeError):
-            self.foo.val

tests/test_threadsafety.py

@@ -0,0 +1,96 @@
+import nilmdb
+from nilmdb.utils.printf import *
+
+import nose
+from nose.tools import *
+from nose.tools import assert_raises
+
+from testutil.helpers import *
+import threading
+
+class Thread(threading.Thread):
+    def __init__(self, target):
+        self.target = target
+        threading.Thread.__init__(self)
+
+    def run(self):
+        try:
+            self.target()
+        except AssertionError as e:
+            self.error = e
+        else:
+            self.error = None
+
+class Test():
+    def __init__(self):
+        self.test = 1234
+
+    @classmethod
+    def asdf(cls):
+        pass
+
+    def foo(self, exception = False, reenter = False):
+        if exception:
+            raise Exception()
+        self.bar(reenter)
+
+    def bar(self, reenter):
+        if reenter:
+            self.foo()
+        return 123
+
+    def baz_threaded(self, target):
+        t = Thread(target)
+        t.start()
+        t.join()
+        return t
+
+    def baz(self, target):
+        target()
+
+class TestThreadSafety(object):
+    def tryit(self, c, threading_ok, concurrent_ok):
+        eq_(c.test, 1234)
+        c.foo()
+        t = Thread(c.foo)
+        t.start()
+        t.join()
+        if threading_ok and t.error:
+            raise Exception("got unexpected error: " + str(t.error))
+        if not threading_ok and not t.error:
+            raise Exception("failed to get expected error")
+        try:
+            c.baz(c.foo)
+        except AssertionError as e:
+            if concurrent_ok:
+                raise Exception("got unexpected error: " + str(e))
+        else:
+            if not concurrent_ok:
+                raise Exception("failed to get expected error")
+        t = c.baz_threaded(c.foo)
+        if (concurrent_ok and threading_ok) and t.error:
+            raise Exception("got unexpected error: " + str(t.error))
+        if not (concurrent_ok and threading_ok) and not t.error:
+            raise Exception("failed to get expected error")
+
+    def test(self):
+        proxy = nilmdb.utils.threadsafety.verify_proxy
+        self.tryit(Test(), True, True)
+        self.tryit(proxy(Test(), True, True, True), False, False)
+        self.tryit(proxy(Test(), True, True, False), False, True)
+        self.tryit(proxy(Test(), True, False, True), True, False)
+        self.tryit(proxy(Test(), True, False, False), True, True)
+        self.tryit(proxy(Test, True, True, True)(), False, False)
+        self.tryit(proxy(Test, True, True, False)(), False, True)
+        self.tryit(proxy(Test, True, False, True)(), True, False)
+        self.tryit(proxy(Test, True, False, False)(), True, True)
+
+        proxy(proxy(proxy(Test))()).foo()
+
+        c = proxy(Test())
+        c.foo()
+        try:
+            c.foo(exception = True)
+        except Exception:
+            pass
+        c.foo()

tests/test_timestamper.py

@@ -1,6 +1,5 @@
 import nilmdb
 from nilmdb.utils.printf import *
-from nilmdb.utils import datetime_tz
 
 from nose.tools import *
 from nose.tools import assert_raises
@@ -20,11 +19,11 @@ class TestTimestamper(object):
         def join(list):
             return "\n".join(list) + "\n"
 
-        start = datetime_tz.datetime_tz.smartparse("03/24/2012").totimestamp()
+        start = nilmdb.utils.time.parse_time("03/24/2012")
         lines_in  = [ "hello", "world", "hello world", "# commented out" ]
-        lines_out = [ "1332561600.000000 hello",
+        lines_out = [ "1332561600000000 hello",
-                      "1332561600.000125 world",
+                      "1332561600000125 world",
-                      "1332561600.000250 hello world" ]
+                      "1332561600000250 hello world" ]
 
         # full
         input = cStringIO.StringIO(join(lines_in))
@@ -42,7 +41,7 @@ class TestTimestamper(object):
         # stop iteration early
         input = cStringIO.StringIO(join(lines_in))
         ts = timestamper.TimestamperRate(input, start, 8000,
-                                                1332561600.000200)
+                                                1332561600000200)
         foo = ""
         for line in ts:
             foo += line
@@ -51,14 +50,14 @@ class TestTimestamper(object):
         # stop iteration early (readlines)
         input = cStringIO.StringIO(join(lines_in))
         ts = timestamper.TimestamperRate(input, start, 8000,
-                                                1332561600.000200)
+                                                1332561600000200)
         foo = ts.readlines()
         eq_(foo, join(lines_out[0:2]))
 
         # stop iteration really early
         input = cStringIO.StringIO(join(lines_in))
         ts = timestamper.TimestamperRate(input, start, 8000,
-                                                1332561600.000000)
+                                                1332561600000000)
         foo = ts.readlines()
         eq_(foo, "")
 
@@ -83,10 +82,3 @@ class TestTimestamper(object):
         for line in ts:
             raise AssertionError
         ts.close()
-
-        # Test the null timestamper
-        input = cStringIO.StringIO(join(lines_out))  # note: lines_out
-        ts = timestamper.TimestamperNull(input)
-        foo = ts.readlines()
-        eq_(foo, join(lines_out))
-        eq_(str(ts), "TimestamperNull(...)")

tests/testutil/aplotter.py

@@ -1,419 +0,0 @@
-
-#-----------------------------------------------
-#aplotter.py - ascii art function plotter
-#Copyright (c) 2006, Imri Goldberg
-#All rights reserved.
-#
-#Redistribution and use in source and binary forms,
-#with or without modification, are permitted provided
-#that the following conditions are met:
-#
-#    * Redistributions of source code must retain the
-#		above copyright notice, this list of conditions
-#		and the following disclaimer.
-#    * Redistributions in binary form must reproduce the
-#		above copyright notice, this list of conditions
-#		and the following disclaimer in the documentation
-#		and/or other materials provided with the distribution.
-#    * Neither the name of the <ORGANIZATION> nor the names of
-#		its contributors may be used to endorse or promote products
-#		derived from this software without specific prior written permission.
-#
-#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-#AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-#IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-#ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-#LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-#DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-#SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-#CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-#OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#-----------------------------------------------
-
-import math
-
-
-EPSILON = 0.000001
-
-def transposed(mat):
-	result = []
-	for i in xrange(len(mat[0])):
-		result.append([x[i] for x in mat])
-	return result
-
-def y_reversed(mat):    
-	result = []
-	for i in range(len(mat)):
-		result.append(list(reversed(mat[i])))
-	return result
-
-def sign(x):
-	if 0<x:
-		return 1
-	if 0 == x:
-		return 0
-	return -1
-
-class Plotter(object):
-
-	class PlotData(object):
-		def __init__(self, x_size, y_size, min_x, max_x, min_y, max_y, x_mod, y_mod):
-			self.x_size = x_size
-			self.y_size = y_size
-			self.min_x = min_x
-			self.max_x = max_x
-			self.min_y = min_y
-			self.max_y = max_y
-			self.x_mod = x_mod
-			self.y_mod = y_mod
-
-			self.x_step = float(max_x - min_x)/float(self.x_size)
-			self.y_step = float(max_y - min_y)/float(self.y_size)
-			self.inv_x_step = 1/self.x_step
-			self.inv_y_step = 1/self.y_step
-
-			self.ratio = self.y_step / self.x_step
-		def __repr__(self):
-			s = "size: %s, bl: %s, tr: %s, step: %s" % ((self.x_size, self.y_size), (self.min_x, self.min_y), (self.max_x, self.max_y),
-														 (self.x_step, self.y_step))
-			return s
-	
-	def __init__(self, **kwargs):
-
-		self.x_size = kwargs.get("x_size", 80)
-		self.y_size = kwargs.get("y_size", 20)
-
-		self.will_draw_axes = kwargs.get("draw_axes", True)
-
-		self.new_line = kwargs.get("newline", "\n")
-
-		self.dot = kwargs.get("dot", "*")
-
-		self.plot_slope = kwargs.get("plot_slope", True)
-
-		self.x_margin = kwargs.get("x_margin", 0.05)
-		self.y_margin = kwargs.get("y_margin", 0.1)
-
-		self.will_plot_labels = kwargs.get("plot_labels", True)
-
-	@staticmethod
-	def get_symbol_by_slope(slope, default_symbol):
-		draw_symbol = default_symbol
-		if slope > math.tan(3*math.pi/8):
-			draw_symbol = "|"
-		elif slope > math.tan(math.pi/8) and slope < math.tan(3*math.pi/8):
-			draw_symbol = "/"
-		elif abs(slope) < math.tan(math.pi/8):
-			draw_symbol = "-"
-		elif slope < math.tan(-math.pi/8) and slope > math.tan(-3*math.pi/8):
-			draw_symbol = "\\"
-		elif slope < math.tan(-3*math.pi/8):
-			draw_symbol = "|"
-		return draw_symbol    
-
-
-	def plot_labels(self, output_buffer, plot_data):
-		if plot_data.y_size < 2:
-			return
-
-		margin_factor = 1
-
-		do_plot_x_label = True
-		do_plot_y_label = True
-
-		x_str = "%+g"
-		if plot_data.x_size < 16:
-			do_plot_x_label = False
-		elif plot_data.x_size < 23:
-			x_str = "%+.2g" 
-
-		y_str = "%+g"    
-		if plot_data.x_size < 8:
-			do_plot_y_label = False
-		elif plot_data.x_size < 11:
-			y_str = "%+.2g"
-			
-		act_min_x = (plot_data.min_x + plot_data.x_mod*margin_factor)
-		act_max_x = (plot_data.max_x - plot_data.x_mod*margin_factor)
-		act_min_y = (plot_data.min_y + plot_data.y_mod*margin_factor)
-		act_max_y = (plot_data.max_y - plot_data.y_mod*margin_factor)
-
-		if abs(act_min_x) < 1:
-			min_x_str = "%+.2g" % act_min_x
-		else:
-			min_x_str = x_str % act_min_x
-
-		if abs(act_max_x) < 1:
-			max_x_str = "%+.2g" % act_max_x
-		else:
-			max_x_str = x_str % act_max_x
-		
-		if abs(act_min_y) < 1:
-			min_y_str = "%+.2g" % act_min_y
-		else:
-			min_y_str = y_str % act_min_y
-
-		if abs(act_max_y) < 1:
-			max_y_str = "%+.2g" % act_max_y
-		else:
-			max_y_str = y_str % act_max_y
-						 
-		min_x_coord = self.get_coord(act_min_x,plot_data.min_x,plot_data.x_step)
-		max_x_coord = self.get_coord(act_max_x,plot_data.min_x,plot_data.x_step)
-		min_y_coord = self.get_coord(act_min_y,plot_data.min_y,plot_data.y_step)
-		max_y_coord = self.get_coord(act_max_y,plot_data.min_y,plot_data.y_step)
-								 
-
-		#print plot_data
-		
-		y_zero_coord = self.get_coord(0, plot_data.min_y, plot_data.y_step)
-
-		#if plot_data.min_x < 0 and plot_data.max_x > 0:
-		x_zero_coord = self.get_coord(0, plot_data.min_x, plot_data.x_step)
-		#else:
-		
-		#pass
-
-		output_buffer[x_zero_coord][min_y_coord] = "+"
-		output_buffer[x_zero_coord][max_y_coord] = "+"
-		output_buffer[min_x_coord][y_zero_coord] = "+"
-		output_buffer[max_x_coord][y_zero_coord] = "+"
-
-		if do_plot_x_label:        
-
-			for i,c in enumerate(min_x_str):
-				output_buffer[min_x_coord+i][y_zero_coord-1] = c
-			for i,c in enumerate(max_x_str):
-				output_buffer[max_x_coord+i-len(max_x_str)][y_zero_coord-1] = c
-
-		if do_plot_y_label:
-
-			for i,c in enumerate(max_y_str):
-				output_buffer[x_zero_coord+i][max_y_coord] = c
-			for i,c in enumerate(min_y_str):
-				output_buffer[x_zero_coord+i][min_y_coord] = c
-			
-
-		
-		
-	
-	def plot_data(self, xy_seq, output_buffer, plot_data):
-		if self.plot_slope:
-			xy_seq = list(xy_seq)
-			#sort according to the x coord
-			xy_seq.sort(key = lambda c: c[0])
-			prev_p = xy_seq[0]
-			e_xy_seq = enumerate(xy_seq)
-			e_xy_seq.next()
-			for i,(x,y) in e_xy_seq:
-				draw_symbol = self.dot
-				line_drawn = self.plot_line(prev_p, (x,y), output_buffer, plot_data)
-				prev_p = (x,y)
-				if not line_drawn:
-					if i > 0 and i < len(xy_seq)-1:
-						px,py = xy_seq[i-1]
-						nx,ny = xy_seq[i+1]
-
-						if abs(nx-px) > EPSILON:
-							slope = (1.0/plot_data.ratio)*(ny-py)/(nx-px)
-							draw_symbol = self.get_symbol_by_slope(slope, draw_symbol)
-					if x < plot_data.min_x or x >= plot_data.max_x or y < plot_data.min_y or y >= plot_data.max_y:
-						continue
-					
-					x_coord = self.get_coord(x, plot_data.min_x, plot_data.x_step)
-					y_coord = self.get_coord(y, plot_data.min_y, plot_data.y_step)            
-					if x_coord >= 0 and x_coord < len(output_buffer) and y_coord >= 0 and y_coord < len(output_buffer[0]):
-						if self.draw_axes:
-							if y_coord == self.get_coord(0, plot_data.min_y, plot_data.y_step) and draw_symbol == "-":
-								draw_symbol = "="
-						output_buffer[x_coord][y_coord] = draw_symbol
-		else:
-			for x,y in xy_seq:
-				if x < plot_data.min_x or x >= plot_data.max_x or y < plot_data.min_y or y >= plot_data.max_y:
-					continue
-				x_coord = self.get_coord(x, plot_data.min_x, plot_data.x_step)
-				y_coord = self.get_coord(y, plot_data.min_y, plot_data.y_step)
-				if x_coord >= 0 and x_coord < len(output_buffer) and y_coord > 0 and y_coord < len(output_buffer[0]):
-					output_buffer[x_coord][y_coord] = self.dot
-
-
-	def plot_line(self, start, end, output_buffer, plot_data):
-
-		start_coord = self.get_coord(start[0], plot_data.min_x, plot_data.x_step), self.get_coord(start[1], plot_data.min_y, plot_data.y_step)
-		end_coord = self.get_coord(end[0], plot_data.min_x, plot_data.x_step), self.get_coord(end[1], plot_data.min_y, plot_data.y_step)
-
-		x0,y0 = start_coord
-		x1,y1 = end_coord
-		if (x0,y0) == (x1,y1):
-			return True    
-		
-		clipped_line = clip_line(start, end, (plot_data.min_x, plot_data.min_y), (plot_data.max_x, plot_data.max_y))
-		if clipped_line != None:
-			start,end = clipped_line
-		else:
-			return False
-		start_coord = self.get_coord(start[0], plot_data.min_x, plot_data.x_step), self.get_coord(start[1], plot_data.min_y, plot_data.y_step)
-		end_coord = self.get_coord(end[0], plot_data.min_x, plot_data.x_step), self.get_coord(end[1], plot_data.min_y, plot_data.y_step)
-
-		x0,y0 = start_coord
-		x1,y1 = end_coord
-		if (x0,y0) == (x1,y1):
-			return True
-		x_zero_coord = self.get_coord(0, plot_data.min_x, plot_data.x_step)
-		y_zero_coord = self.get_coord(0, plot_data.min_y, plot_data.y_step)   
-
-		if start[0]-end[0] == 0:
-			draw_symbol = "|"
-		else:
-			slope = (1.0/plot_data.ratio)*(end[1]-start[1])/(end[0]-start[0])
-			draw_symbol = self.get_symbol_by_slope(slope, self.dot)
-		try:
-
-			delta = x1-x0, y1-y0
-			if abs(delta[0])>abs(delta[1]):
-				s = sign(delta[0])
-				slope = float(delta[1])/delta[0]
-				for i in range(0,abs(int(delta[0]))):
-					cur_draw_symbol = draw_symbol
-					x = i*s
-					cur_y = int(y0+slope*x)
-					if self.draw_axes and cur_y == y_zero_coord and draw_symbol == "-":
-						cur_draw_symbol = "="
-					output_buffer[x0+x][cur_y] = cur_draw_symbol
-				
-				
-			else:
-				s = sign(delta[1])
-				slope = float(delta[0])/delta[1]
-				for i in range(0,abs(int(delta[1]))):
-					y = i*s
-					cur_draw_symbol = draw_symbol
-					cur_y = y0+y
-					if self.draw_axes and cur_y == y_zero_coord and draw_symbol == "-":
-						cur_draw_symbol = "="
-					output_buffer[int(x0+slope*y)][cur_y] = cur_draw_symbol
-		except:
-			print start, end
-			print start_coord, end_coord
-			print plot_data
-			raise
-
-		return False            
-		
-		
-	def plot_single(self, seq, min_x = None, max_x = None, min_y = None, max_y = None):
-		return self.plot_double(range(len(seq)),seq, min_x, max_x, min_y, max_y)
-		
-
-
-
-	def plot_double(self, x_seq, y_seq, min_x = None, max_x = None, min_y = None, max_y = None):
-		if min_x == None:
-			min_x = min(x_seq)
-		if max_x == None:
-			max_x = max(x_seq)
-		if min_y == None:
-			min_y = min(y_seq)
-		if max_y == None:
-			max_y = max(y_seq)
-
-		if max_y == min_y:
-			max_y += 1
-
-		x_mod = (max_x-min_x)*self.x_margin
-		y_mod = (max_y-min_y)*self.y_margin
-		min_x-=x_mod
-		max_x+=x_mod
-		min_y-=y_mod
-		max_y+=y_mod
-
-
-		plot_data = self.PlotData(self.x_size, self.y_size, min_x, max_x, min_y, max_y, x_mod, y_mod)
-
-		output_buffer = [[" "]*self.y_size for i in range(self.x_size)]
-
-		if self.will_draw_axes:
-			self.draw_axes(output_buffer, plot_data)
-
-		self.plot_data(zip(x_seq, y_seq), output_buffer, plot_data)
-
-		if self.will_plot_labels:
-			self.plot_labels(output_buffer, plot_data)
-
-		trans_result = transposed(y_reversed(output_buffer))
-
-		result = self.new_line.join(["".join(row) for row in trans_result])
-		return result
-
-	def draw_axes(self, output_buffer, plot_data):
-		
-		
-		draw_x = False
-		draw_y = False
-
-		if plot_data.min_x <= 0 and plot_data.max_x > 0:
-			draw_y = True
-			zero_x = self.get_coord(0, plot_data.min_x, plot_data.x_step)
-			for y in xrange(plot_data.y_size):
-				output_buffer[zero_x][y] = "|"
-				
-		if plot_data.min_y <= 0 and plot_data.max_y > 0:
-			draw_x = True
-			zero_y = self.get_coord(0, plot_data.min_y, plot_data.y_step)    
-			for x in xrange(plot_data.x_size):
-				output_buffer[x][zero_y] = "-"
-
-		if draw_x and draw_y:
-			output_buffer[zero_x][zero_y] = "+"
-		
-		
-	@staticmethod
-	def get_coord(val, min, step):
-		result = int((val - min)/step)
-		return result
-
-def clip_line(line_pt_1, line_pt_2, rect_bottom_left, rect_top_right):
-	ts = [0.0,1.0]
-	if line_pt_1[0] == line_pt_2[0]:
-		return ((line_pt_1[0], max(min(line_pt_1[1], line_pt_2[1]), rect_bottom_left[1])),
-				(line_pt_1[0], min(max(line_pt_1[1], line_pt_2[1]), rect_top_right[1])))
-	if line_pt_1[1] == line_pt_2[1]:
-		return ((max(min(line_pt_1[0], line_pt_2[0]), rect_bottom_left[0]), line_pt_1[1]),
-				(min(max(line_pt_1[0], line_pt_2[0]), rect_top_right[0]), line_pt_1[1]))
-
-	if ((rect_bottom_left[0] <= line_pt_1[0] and line_pt_1[0] < rect_top_right[0]) and
-		(rect_bottom_left[1] <= line_pt_1[1] and line_pt_1[1] < rect_top_right[1]) and
-		(rect_bottom_left[0] <= line_pt_2[0] and line_pt_2[0] < rect_top_right[0]) and
-		(rect_bottom_left[1] <= line_pt_2[1] and line_pt_2[1] < rect_top_right[1])):
-		return line_pt_1, line_pt_2
-
-	ts.append( float(rect_bottom_left[0]-line_pt_1[0])/(line_pt_2[0]-line_pt_1[0]) )
-	ts.append( float(rect_top_right[0]-line_pt_1[0])/(line_pt_2[0]-line_pt_1[0]) )
-	ts.append( float(rect_bottom_left[1]-line_pt_1[1])/(line_pt_2[1]-line_pt_1[1]) )
-	ts.append( float(rect_top_right[1]-line_pt_1[1])/(line_pt_2[1]-line_pt_1[1]) )
-	
-	ts.sort()
-	if ts[2] < 0 or ts[2] >= 1 or ts[3] < 0 or ts[2]>= 1:
-		return None
-	result = [(pt_1 + t*(pt_2-pt_1)) for t in (ts[2],ts[3]) for (pt_1, pt_2) in zip(line_pt_1, line_pt_2)]
-	return (result[0],result[1]), (result[2], result[3])
-	
-
-
-def plot(*args,**flags):
-	limit_flags_names = set(["min_x","min_y","max_x","max_y"])
-	limit_flags = dict([(n,flags[n]) for n in limit_flags_names & set(flags)])
-	settting_flags = dict([(n,flags[n]) for n in set(flags) - limit_flags_names])
-	
-	if len(args) == 1:
-		p = Plotter(**settting_flags)
-		print p.plot_single(args[0],**limit_flags)
-	elif len(args) == 2:
-		p = Plotter(**settting_flags)
-		print p.plot_double(args[0],args[1],**limit_flags)
-	else:
-		raise NotImplementedError("can't draw multiple graphs yet")
-	
-__all__ = ["Plotter","plot"]
-

tests/testutil/helpers.py

@@ -20,6 +20,11 @@ def in_(a, b):
     if a not in b:
         raise AssertionError("%s not in %s" % (myrepr(a), myrepr(b)))
 
+def in2_(a1, a2, b):
+    if a1 not in b and a2 not in b:
+        raise AssertionError("(%s or %s) not in %s" % (myrepr(a1), myrepr(a2),
+                                                       myrepr(b)))
+
 def ne_(a, b):
     if not a != b:
         raise AssertionError("unexpected %s == %s" % (myrepr(a), myrepr(b)))

versioneer.py

@@ -0,0 +1,655 @@
+#! /usr/bin/python
+
+"""versioneer.py
+
+(like a rocketeer, but for versions)
+
+* https://github.com/warner/python-versioneer
+* Brian Warner
+* License: Public Domain
+* Version: 0.7+
+
+This file helps distutils-based projects manage their version number by just
+creating version-control tags.
+
+For developers who work from a VCS-generated tree (e.g. 'git clone' etc),
+each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a
+version number by asking your version-control tool about the current
+checkout. The version number will be written into a generated _version.py
+file of your choosing, where it can be included by your __init__.py
+
+For users who work from a VCS-generated tarball (e.g. 'git archive'), it will
+compute a version number by looking at the name of the directory created when
+te tarball is unpacked. This conventionally includes both the name of the
+project and a version number.
+
+For users who work from a tarball built by 'setup.py sdist', it will get a
+version number from a previously-generated _version.py file.
+
+As a result, loading code directly from the source tree will not result in a
+real version. If you want real versions from VCS trees (where you frequently
+update from the upstream repository, or do new development), you will need to
+do a 'setup.py version' after each update, and load code from the build/
+directory.
+
+You need to provide this code with a few configuration values:
+
+ versionfile_source:
+    A project-relative pathname into which the generated version strings
+    should be written. This is usually a _version.py next to your project's
+    main __init__.py file. If your project uses src/myproject/__init__.py,
+    this should be 'src/myproject/_version.py'. This file should be checked
+    in to your VCS as usual: the copy created below by 'setup.py
+    update_files' will include code that parses expanded VCS keywords in
+    generated tarballs. The 'build' and 'sdist' commands will replace it with
+    a copy that has just the calculated version string.
+
+ versionfile_build:
+    Like versionfile_source, but relative to the build directory instead of
+    the source directory. These will differ when your setup.py uses
+    'package_dir='. If you have package_dir={'myproject': 'src/myproject'},
+    then you will probably have versionfile_build='myproject/_version.py' and
+    versionfile_source='src/myproject/_version.py'.
+
+ tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all
+             VCS tags. If your tags look like 'myproject-1.2.0', then you
+             should use tag_prefix='myproject-'. If you use unprefixed tags
+             like '1.2.0', this should be an empty string.
+
+ parentdir_prefix: a string, frequently the same as tag_prefix, which
+                   appears at the start of all unpacked tarball filenames. If
+                   your tarball unpacks into 'myproject-1.2.0', this should
+                   be 'myproject-'.
+
+To use it:
+
+ 1: include this file in the top level of your project
+ 2: make the following changes to the top of your setup.py:
+     import versioneer
+     versioneer.versionfile_source = 'src/myproject/_version.py'
+     versioneer.versionfile_build = 'myproject/_version.py'
+     versioneer.tag_prefix = '' # tags are like 1.2.0
+     versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0'
+ 3: add the following arguments to the setup() call in your setup.py:
+     version=versioneer.get_version(),
+     cmdclass=versioneer.get_cmdclass(),
+ 4: run 'setup.py update_files', which will create _version.py, and will
+    append the following to your __init__.py:
+     from _version import __version__
+ 5: modify your MANIFEST.in to include versioneer.py
+ 6: add both versioneer.py and the generated _version.py to your VCS
+"""
+
+import os, sys, re
+from distutils.core import Command
+from distutils.command.sdist import sdist as _sdist
+from distutils.command.build_py import build_py as _build_py
+
+versionfile_source = None
+versionfile_build = None
+tag_prefix = None
+parentdir_prefix = None
+
+VCS = "git"
+IN_LONG_VERSION_PY = False
+
+
+LONG_VERSION_PY = '''
+IN_LONG_VERSION_PY = True
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by githubs download-from-tag
+# feature). Distribution tarballs (build by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
+
+# This file is released into the public domain. Generated by
+# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+
+# these strings will be replaced by git during git-archive
+git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
+git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+
+
+import subprocess
+import sys
+
+def run_command(args, cwd=None, verbose=False):
+    try:
+        # remember shell=False, so use git.cmd on windows, not just git
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
+    except EnvironmentError:
+        e = sys.exc_info()[1]
+        if verbose:
+            print("unable to run %%s" %% args[0])
+            print(e)
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version >= '3':
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %%s (error)" %% args[0])
+        return None
+    return stdout
+
+
+import sys
+import re
+import os.path
+
+def get_expanded_variables(versionfile_source):
+    # the code embedded in _version.py can just fetch the value of these
+    # variables. When used from setup.py, we don't want to import
+    # _version.py, so we do it with a regexp instead. This function is not
+    # used from _version.py.
+    variables = {}
+    try:
+        for line in open(versionfile_source,"r").readlines():
+            if line.strip().startswith("git_refnames ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["refnames"] = mo.group(1)
+            if line.strip().startswith("git_full ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["full"] = mo.group(1)
+    except EnvironmentError:
+        pass
+    return variables
+
+def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
+    refnames = variables["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("variables are unexpanded, not using")
+        return {} # unexpanded, so not in an unpacked git-archive tarball
+    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    for ref in list(refs):
+        if not re.search(r'\d', ref):
+            if verbose:
+                print("discarding '%%s', no digits" %% ref)
+            refs.discard(ref)
+            # Assume all version tags have a digit. git's %%d expansion
+            # behaves like git log --decorate=short and strips out the
+            # refs/heads/ and refs/tags/ prefixes that would let us
+            # distinguish between branches and tags. By ignoring refnames
+            # without digits, we filter out many common branch names like
+            # "release" and "stabilization", as well as "HEAD" and "master".
+    if verbose:
+        print("remaining refs: %%s" %% ",".join(sorted(refs)))
+    for ref in sorted(refs):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix):]
+            if verbose:
+                print("picking %%s" %% r)
+            return { "version": r,
+                     "full": variables["full"].strip() }
+    # no suitable tags, so we use the full revision id
+    if verbose:
+        print("no suitable tags, using full revision id")
+    return { "version": variables["full"].strip(),
+             "full": variables["full"].strip() }
+
+def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
+    # this runs 'git' from the root of the source tree. That either means
+    # someone ran a setup.py command (and this code is in versioneer.py, so
+    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
+    # the source tree), or someone ran a project-specific entry point (and
+    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
+    # containing directory is somewhere deeper in the source tree). This only
+    # gets called if the git-archive 'subst' variables were *not* expanded,
+    # and _version.py hasn't already been rewritten with a short version
+    # string, meaning we're inside a checked out source tree.
+
+    try:
+        here = os.path.abspath(__file__)
+    except NameError:
+        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
+        return {} # not always correct
+
+    # versionfile_source is the relative path from the top of the source tree
+    # (where the .git directory might live) to this file. Invert this to find
+    # the root from __file__.
+    root = here
+    if IN_LONG_VERSION_PY:
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        root = os.path.dirname(here)
+    if not os.path.exists(os.path.join(root, ".git")):
+        if verbose:
+            print("no .git in %%s" %% root)
+        return {}
+
+    GIT = "git"
+    if sys.platform == "win32":
+        GIT = "git.cmd"
+    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
+                         cwd=root)
+    if stdout is None:
+        return {}
+    if not stdout.startswith(tag_prefix):
+        if verbose:
+            print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix))
+        return {}
+    tag = stdout[len(tag_prefix):]
+    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
+    if stdout is None:
+        return {}
+    full = stdout.strip()
+    if tag.endswith("-dirty"):
+        full += "-dirty"
+    return {"version": tag, "full": full}
+
+
+def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
+    if IN_LONG_VERSION_PY:
+        # We're running from _version.py. If it's from a source tree
+        # (execute-in-place), we can work upwards to find the root of the
+        # tree, and then check the parent directory for a version string. If
+        # it's in an installed application, there's no hope.
+        try:
+            here = os.path.abspath(__file__)
+        except NameError:
+            # py2exe/bbfreeze/non-CPython don't have __file__
+            return {} # without __file__, we have no hope
+        # versionfile_source is the relative path from the top of the source
+        # tree to _version.py. Invert this to find the root from __file__.
+        root = here
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        # we're running from versioneer.py, which means we're running from
+        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
+        here = os.path.abspath(sys.argv[0])
+        root = os.path.dirname(here)
+
+    # Source tarballs conventionally unpack into a directory that includes
+    # both the project name and a version string.
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %%
+                  (root, dirname, parentdir_prefix))
+        return None
+    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+
+tag_prefix = "%(TAG_PREFIX)s"
+parentdir_prefix = "%(PARENTDIR_PREFIX)s"
+versionfile_source = "%(VERSIONFILE_SOURCE)s"
+
+def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
+    variables = { "refnames": git_refnames, "full": git_full }
+    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
+    if not ver:
+        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
+    if not ver:
+        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
+                                      verbose)
+    if not ver:
+        ver = default
+    return ver
+
+'''
+
+
+import subprocess
+import sys
+
+def run_command(args, cwd=None, verbose=False):
+    try:
+        # remember shell=False, so use git.cmd on windows, not just git
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
+    except EnvironmentError:
+        e = sys.exc_info()[1]
+        if verbose:
+            print("unable to run %s" % args[0])
+            print(e)
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version >= '3':
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % args[0])
+        return None
+    return stdout
+
+
+import sys
+import re
+import os.path
+
+def get_expanded_variables(versionfile_source):
+    # the code embedded in _version.py can just fetch the value of these
+    # variables. When used from setup.py, we don't want to import
+    # _version.py, so we do it with a regexp instead. This function is not
+    # used from _version.py.
+    variables = {}
+    try:
+        for line in open(versionfile_source,"r").readlines():
+            if line.strip().startswith("git_refnames ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["refnames"] = mo.group(1)
+            if line.strip().startswith("git_full ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["full"] = mo.group(1)
+    except EnvironmentError:
+        pass
+    return variables
+
+def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
+    refnames = variables["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("variables are unexpanded, not using")
+        return {} # unexpanded, so not in an unpacked git-archive tarball
+    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    for ref in list(refs):
+        if not re.search(r'\d', ref):
+            if verbose:
+                print("discarding '%s', no digits" % ref)
+            refs.discard(ref)
+            # Assume all version tags have a digit. git's %d expansion
+            # behaves like git log --decorate=short and strips out the
+            # refs/heads/ and refs/tags/ prefixes that would let us
+            # distinguish between branches and tags. By ignoring refnames
+            # without digits, we filter out many common branch names like
+            # "release" and "stabilization", as well as "HEAD" and "master".
+    if verbose:
+        print("remaining refs: %s" % ",".join(sorted(refs)))
+    for ref in sorted(refs):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix):]
+            if verbose:
+                print("picking %s" % r)
+            return { "version": r,
+                     "full": variables["full"].strip() }
+    # no suitable tags, so we use the full revision id
+    if verbose:
+        print("no suitable tags, using full revision id")
+    return { "version": variables["full"].strip(),
+             "full": variables["full"].strip() }
+
+def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
+    # this runs 'git' from the root of the source tree. That either means
+    # someone ran a setup.py command (and this code is in versioneer.py, so
+    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
+    # the source tree), or someone ran a project-specific entry point (and
+    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
+    # containing directory is somewhere deeper in the source tree). This only
+    # gets called if the git-archive 'subst' variables were *not* expanded,
+    # and _version.py hasn't already been rewritten with a short version
+    # string, meaning we're inside a checked out source tree.
+
+    try:
+        here = os.path.abspath(__file__)
+    except NameError:
+        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
+        return {} # not always correct
+
+    # versionfile_source is the relative path from the top of the source tree
+    # (where the .git directory might live) to this file. Invert this to find
+    # the root from __file__.
+    root = here
+    if IN_LONG_VERSION_PY:
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        root = os.path.dirname(here)
+    if not os.path.exists(os.path.join(root, ".git")):
+        if verbose:
+            print("no .git in %s" % root)
+        return {}
+
+    GIT = "git"
+    if sys.platform == "win32":
+        GIT = "git.cmd"
+    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
+                         cwd=root)
+    if stdout is None:
+        return {}
+    if not stdout.startswith(tag_prefix):
+        if verbose:
+            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
+        return {}
+    tag = stdout[len(tag_prefix):]
+    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
+    if stdout is None:
+        return {}
+    full = stdout.strip()
+    if tag.endswith("-dirty"):
+        full += "-dirty"
+    return {"version": tag, "full": full}
+
+
+def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
+    if IN_LONG_VERSION_PY:
+        # We're running from _version.py. If it's from a source tree
+        # (execute-in-place), we can work upwards to find the root of the
+        # tree, and then check the parent directory for a version string. If
+        # it's in an installed application, there's no hope.
+        try:
+            here = os.path.abspath(__file__)
+        except NameError:
+            # py2exe/bbfreeze/non-CPython don't have __file__
+            return {} # without __file__, we have no hope
+        # versionfile_source is the relative path from the top of the source
+        # tree to _version.py. Invert this to find the root from __file__.
+        root = here
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        # we're running from versioneer.py, which means we're running from
+        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
+        here = os.path.abspath(sys.argv[0])
+        root = os.path.dirname(here)
+
+    # Source tarballs conventionally unpack into a directory that includes
+    # both the project name and a version string.
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
+                  (root, dirname, parentdir_prefix))
+        return None
+    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+
+import sys
+
+def do_vcs_install(versionfile_source, ipy):
+    GIT = "git"
+    if sys.platform == "win32":
+        GIT = "git.cmd"
+    run_command([GIT, "add", "versioneer.py"])
+    run_command([GIT, "add", versionfile_source])
+    run_command([GIT, "add", ipy])
+    present = False
+    try:
+        f = open(".gitattributes", "r")
+        for line in f.readlines():
+            if line.strip().startswith(versionfile_source):
+                if "export-subst" in line.strip().split()[1:]:
+                    present = True
+        f.close()
+    except EnvironmentError:
+        pass    
+    if not present:
+        f = open(".gitattributes", "a+")
+        f.write("%s export-subst\n" % versionfile_source)
+        f.close()
+        run_command([GIT, "add", ".gitattributes"])
+    
+
+SHORT_VERSION_PY = """
+# This file was generated by 'versioneer.py' (0.7+) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '%(version)s'
+version_full = '%(full)s'
+def get_versions(default={}, verbose=False):
+    return {'version': version_version, 'full': version_full}
+
+"""
+
+DEFAULT = {"version": "unknown", "full": "unknown"}
+
+def versions_from_file(filename):
+    versions = {}
+    try:
+        f = open(filename)
+    except EnvironmentError:
+        return versions
+    for line in f.readlines():
+        mo = re.match("version_version = '([^']+)'", line)
+        if mo:
+            versions["version"] = mo.group(1)
+        mo = re.match("version_full = '([^']+)'", line)
+        if mo:
+            versions["full"] = mo.group(1)
+    return versions
+
+def write_to_version_file(filename, versions):
+    f = open(filename, "w")
+    f.write(SHORT_VERSION_PY % versions)
+    f.close()
+    print("set %s to '%s'" % (filename, versions["version"]))
+
+
+def get_best_versions(versionfile, tag_prefix, parentdir_prefix,
+                      default=DEFAULT, verbose=False):
+    # returns dict with two keys: 'version' and 'full'
+    #
+    # extract version from first of _version.py, 'git describe', parentdir.
+    # This is meant to work for developers using a source checkout, for users
+    # of a tarball created by 'setup.py sdist', and for users of a
+    # tarball/zipball created by 'git archive' or github's download-from-tag
+    # feature.
+
+    variables = get_expanded_variables(versionfile_source)
+    if variables:
+        ver = versions_from_expanded_variables(variables, tag_prefix)
+        if ver:
+            if verbose: print("got version from expanded variable %s" % ver)
+            return ver
+
+    ver = versions_from_file(versionfile)
+    if ver:
+        if verbose: print("got version from file %s %s" % (versionfile, ver))
+        return ver
+
+    ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
+    if ver:
+        if verbose: print("got version from git %s" % ver)
+        return ver
+
+    ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose)
+    if ver:
+        if verbose: print("got version from parentdir %s" % ver)
+        return ver
+
+    if verbose: print("got version from default %s" % ver)
+    return default
+
+def get_versions(default=DEFAULT, verbose=False):
+    assert versionfile_source is not None, "please set versioneer.versionfile_source"
+    assert tag_prefix is not None, "please set versioneer.tag_prefix"
+    assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix"
+    return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix,
+                             default=default, verbose=verbose)
+def get_version(verbose=False):
+    return get_versions(verbose=verbose)["version"]
+
+class cmd_version(Command):
+    description = "report generated version string"
+    user_options = []
+    boolean_options = []
+    def initialize_options(self):
+        pass
+    def finalize_options(self):
+        pass
+    def run(self):
+        ver = get_version(verbose=True)
+        print("Version is currently: %s" % ver)
+
+class cmd_build_py(_build_py):
+    def run(self):
+        versions = get_versions(verbose=True)
+        _build_py.run(self)
+        # now locate _version.py in the new build/ directory and replace it
+        # with an updated value
+        target_versionfile = os.path.join(self.build_lib, versionfile_build)
+        print("UPDATING %s" % target_versionfile)
+        os.unlink(target_versionfile)
+        f = open(target_versionfile, "w")
+        f.write(SHORT_VERSION_PY % versions)
+        f.close()
+
+class cmd_sdist(_sdist):
+    def run(self):
+        versions = get_versions(verbose=True)
+        self._versioneer_generated_versions = versions
+        # unless we update this, the command will keep using the old version
+        self.distribution.metadata.version = versions["version"]
+        return _sdist.run(self)
+
+    def make_release_tree(self, base_dir, files):
+        _sdist.make_release_tree(self, base_dir, files)
+        # now locate _version.py in the new base_dir directory (remembering
+        # that it may be a hardlink) and replace it with an updated value
+        target_versionfile = os.path.join(base_dir, versionfile_source)
+        print("UPDATING %s" % target_versionfile)
+        os.unlink(target_versionfile)
+        f = open(target_versionfile, "w")
+        f.write(SHORT_VERSION_PY % self._versioneer_generated_versions)
+        f.close()
+
+INIT_PY_SNIPPET = """
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
+"""
+
+class cmd_update_files(Command):
+    description = "modify __init__.py and create _version.py"
+    user_options = []
+    boolean_options = []
+    def initialize_options(self):
+        pass
+    def finalize_options(self):
+        pass
+    def run(self):
+        ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py")
+        print(" creating %s" % versionfile_source)
+        f = open(versionfile_source, "w")
+        f.write(LONG_VERSION_PY % {"DOLLAR": "$",
+                                   "TAG_PREFIX": tag_prefix,
+                                   "PARENTDIR_PREFIX": parentdir_prefix,
+                                   "VERSIONFILE_SOURCE": versionfile_source,
+                                   })
+        f.close()
+        try:
+            old = open(ipy, "r").read()
+        except EnvironmentError:
+            old = ""
+        if INIT_PY_SNIPPET not in old:
+            print(" appending to %s" % ipy)
+            f = open(ipy, "a")
+            f.write(INIT_PY_SNIPPET)
+            f.close()
+        else:
+            print(" %s unmodified" % ipy)
+        do_vcs_install(versionfile_source, ipy)
+
+def get_cmdclass():
+    return {'version': cmd_version,
+            'update_files': cmd_update_files,
+            'build_py': cmd_build_py,
+            'sdist': cmd_sdist,
+            }