server: consolidate time processing and checks

This is instead of caching it inside nilmdb.server.wsgi_application.
Might make things work a bit better in case the web server decides
to call wsgi_application multiple times.

.coveragerc

@@ -7,3 +7,4 @@
 exclude_lines =
 	pragma: no cover
 	if 0:
+omit = nilmdb/utils/datetime_tz*,nilmdb/scripts,nilmdb/_version.py

.gitattributes

@@ -0,0 +1 @@
+nilmdb/_version.py export-subst

.gitignore

@@ -1,4 +1,25 @@
-db/
+# Tests
 tests/*testdb/
 .coverage
+db/
+
+# Compiled / cythonized files
+docs/*.html
+build/
 *.pyc
+nilmdb/server/interval.c
+nilmdb/server/layout.c
+nilmdb/server/rbtree.c
+*.so
+
+# Setup junk
+dist/
+nilmdb.egg-info/
+
+# This gets generated as needed by setup.py
+MANIFEST.in
+MANIFEST
+
+# Misc
+timeit*out
+

.pylintrc

@@ -0,0 +1,250 @@
+# -*- conf -*-
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=datetime_tz
+
+# Pickle collected data for later comparisons.
+persistent=no
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+disable=C0111,R0903,R0201,R0914,R0912,W0142,W0703,W0702
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html
+output-format=parseable
+
+# Include message's id in output
+include-ids=yes
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (RP0004).
+comment=no
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, add a predefined set of Zope acquired attributes
+# to generated-members.
+zope=no
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E0201 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=80
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the beginning of the name of dummy variables
+# (i.e. not used).
+dummy-variables-rgx=_|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=apply,input
+
+# Regular expression which should only match correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression which should only match correct module level names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|version)$
+
+# Regular expression which should only match correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression which should only match correct function names
+function-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct method names
+method-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct argument names
+argument-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct variable names
+variable-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Regular expression which should only match functions or classes name which do
+# not require a docstring
+no-docstring-rgx=__.*__
+
+
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defines in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branchs=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception

Makefile

@@ -1,20 +1,46 @@
+# By default, run the tests.
 all: test
 
-tool:
-	python nilmtool.py  --help
-	python nilmtool.py list --help
-	python nilmtool.py -u asfdadsf list
+version:
+	python setup.py version
+
+build:
+	python setup.py build_ext --inplace
+
+dist: sdist
+sdist:
+	python setup.py sdist
+
+install:
+	python setup.py install
+
+develop:
+	python setup.py develop
+
+docs:
+	make -C docs
 
 lint:
-	pylint -f parseable nilmdb
+	pylint --rcfile=.pylintrc nilmdb
 
 test:
-	nosetests
-
-profile:
-	nosetests --with-profile 
+ifeq ($(INSIDE_EMACS), t)
+# Use the slightly more flexible script
+	python setup.py build_ext --inplace
+	python tests/runtests.py
+else
+# Let setup.py check dependencies, build stuff, and run the test
+	python setup.py nosetests
+endif
 
 clean::
 	find . -name '*pyc' | xargs rm -f
 	rm -f .coverage
 	rm -rf tests/*testdb*
+	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so MANIFEST.in
+	make -C docs clean
+
+gitclean::
+	git clean -dXf
+
+.PHONY: all version build dist sdist install docs lint test clean gitclean

README.txt

@@ -1,4 +1,28 @@
-sudo apt-get install python-nose python-coverage
-sudo apt-get install python-tables python-cherrypy3
-sudo apt-get install cython # 0.17.1-1 or newer
+nilmdb: Non-Intrusive Load Monitor Database
+by Jim Paris <jim@jtan.com>
 
+Prerequisites:
+
+  # Runtime and build environments
+  sudo apt-get install python2.7 python2.7-dev python-setuptools cython
+
+  # Base NilmDB dependencies
+  sudo apt-get install python-cherrypy3 python-decorator python-simplejson
+  sudo apt-get install python-requests python-dateutil python-tz python-psutil
+
+  # Tools for running tests
+  sudo apt-get install python-nose python-coverage
+
+Test:
+  python setup.py nosetests
+
+Install:
+
+  python setup.py install
+
+Usage:
+
+  nilmdb-server --help
+  nilmtool --help
+
+See docs/wsgi.md for info on setting up a WSGI application in Apache.

TODO

@@ -1 +0,0 @@
-- Merge adjacent intervals on insert (maybe with client help?)

design.md

@@ -1,200 +0,0 @@
-Structure
----------
-nilmdb.nilmdb is the NILM database interface.  It tracks a PyTables
-database holds actual rows of data, and a SQL database tracks metadata
-and ranges.
-
-Access to the nilmdb must be single-threaded.  This is handled with
-the nilmdb.serializer class.
-
-nilmdb.server is a HTTP server that provides an interface to talk,
-thorugh the serialization layer, to the nilmdb object.
-
-nilmdb.client is a HTTP client that connects to this.
-
-Sqlite performance
-------------------
-
-Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
-takes about 125msec.  sqlite3 will commit transactions at 3 times:
-
-  1: explicit con.commit()
-
-  2: between a series of DML commands and non-DML commands, e.g.
-   after a series of INSERT, SELECT, but before a CREATE TABLE or
-   PRAGMA.
-
-3: at the end of an explicit transaction, e.g. "with self.con as con:"
-
-To speed up testing, or if this transaction speed becomes an issue,
-the sync=False option to NilmDB will set PRAGMA synchronous=OFF.
-
-
-Inserting streams
------------------
-
-We need to send the contents of "data" as POST.  Do we need chunked
-transfer?
-
-- Don't know the size in advance, so we would need to use chunked if
-  we send the entire thing in one request.
-- But we shouldn't send one chunk per line, so we need to buffer some
-  anyway; why not just make new requests?
-- Consider the infinite-streaming case, we might want to send it
-  immediately?  Not really -- server still should do explicit inserts
-  of fixed-size chunks.
-- Even chunked encoding needs the size of each chunk beforehand, so
-  everything still gets buffered.  Just a tradeoff of buffer size.
-
-Before timestamps are added:
-- Raw data is about 440 kB/s    (9 channels)
-- Prep data is about 12.5 kB/s  (1 phase)
-- How do we know how much data to send?
-
-  - Remember that we can only do maybe 8-50 transactions per second on
-    the sqlite database.  So if one block of inserted data is one
-    transaction, we'd need the raw case to be around 64kB per request,
-	ideally more.
-  - Maybe use a range, based on how long it's taking to read the data
-    - If no more data, send it
-    - If data > 1 MB, send it
-	- If more than 10 seconds have elapsed, send it
-  - Should those numbers come from the server?
-
-Converting from ASCII to PyTables:
-- For each row getting added, we need to set attributes on a PyTables
-  Row object and call table.append().  This means that there isn't a
-  particularly efficient way of converting from ascii.
-- Could create a function like nilmdb.layout.Layout("foo".fillRow(asciiline)
-  - But this means we're doing parsing on the serialized side
-  - Let's keep parsing on the threaded server side so we can detect
-    errors better, and not block the serialized nilmdb for a slow
-    parsing process.
-- Client sends ASCII data
-- Server converts this ACSII data to a list of values
-  - Maybe:
-
-		# threaded side creates this object
-		parser = nilmdb.layout.Parser("layout_name")
-		# threaded side parses and fills it with data
-		parser.parse(textdata)
-		# serialized side pulls out rows
-		for n in xrange(parser.nrows):
-		    parser.fill_row(rowinstance, n)
-			table.append()
-
-
-Inserting streams, inside nilmdb
---------------------------------
-
-- First check that the new stream doesn't overlap.
-  - Get minimum timestamp, maximum timestamp from data parser.
-    - (extend parser to verify monotonicity and track extents)
-  - Get all intervals for this stream in the database
-  - See if new interval overlaps any existing ones
-    - If so, bail
-  - Question: should we cache intervals inside NilmDB?
-    - Assume database is fast for now, and always rebuild fom DB.
-	- Can add a caching layer later if we need to.
-  - `stream_get_ranges(path)` -> return IntervalSet?
-
-Speed
------
-
-- First approach was quadratic.  Adding four hours of data:
-
-    $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/1/raw
-	real    24m31.093s
-	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/1/raw
-	real    43m44.528s
-	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-130002 /bpnilm/1/raw
-	real    93m29.713s
-	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-140003 /bpnilm/1/raw
-	real    166m53.007s
-
-- Disabling pytables indexing didn't help:
-
-    real    31m21.492s
-	real    52m51.963s
-	real    102m8.151s
-	real    176m12.469s
-
-- Server RAM usage is constant.
-
-- Speed problems were due to IntervalSet speed, of parsing intervals
-  from the database and adding the new one each time.
-
-  - First optimization is to cache result of `nilmdb:_get_intervals`,
-    which gives the best speedup.
-
-  - Also switched to internally using bxInterval from bx-python package.
-    Speed of `tests/test_interval:TestIntervalSpeed` is pretty decent
-	and seems to be growing logarithmically now.  About 85μs per insertion
-	for inserting 131k entries.
-
-  - Storing the interval data in SQL might be better, with a scheme like:
-    http://www.logarithmic.net/pfh/blog/01235197474
-
-- Next slowdown target is nilmdb.layout.Parser.parse().
-  - Rewrote parsers using cython and sscanf
-  - Stats (rev 10831), with _add_interval disabled
-     layout.pyx.Parser.parse:128        6303 sec, 262k calls
-	 layout.pyx.parse:63               13913 sec, 5.1g calls
-	 numpy:records.py.fromrecords:569   7410 sec, 262k calls
-  - Probably OK for now.
-
-- After all updates, now takes about 8.5 minutes to insert an hour of
-  data, constant after adding 171 hours (4.9 billion data points)
-
-- Data set size: 98 gigs = 20 bytes per data point.
-  6 uint16 data + 1 uint32 timestamp = 16 bytes per point
-  So compression must be off -- will retry with compression forced on.
-
-IntervalSet speed
------------------
-- Initial implementation was pretty slow, even with binary search in
-  sorted list
-
-- Replaced with bxInterval; now takes about log n time for an insertion
-  - TestIntervalSpeed with range(17,18) and profiling
-    - 85 μs each
-    - 131072 calls to `__iadd__`
-    - 131072 to bx.insert_interval
-    - 131072 to bx.insert:395
-    - 2355835 to bx.insert:106  (18x as many?)
-
-- Tried blist too, worse than bxinterval.
-
-- Might be algorithmic improvements to be made in Interval.py,
-  like in `__and__`
-
-- Replaced again with rbtree.  Seems decent.  Numbers are time per
-  insert for 2**17 insertions, followed by total wall time and RAM
-  usage for running "make test" with `test_rbtree` and `test_interval`
-  with range(5,20):
-  - old values with bxinterval:
-    20.2 μS, total 20 s, 177 MB RAM
-  - rbtree, plain python:
-    97 μS, total 105 s, 846 MB RAM
-  - rbtree converted to cython:
-    26 μS, total 29 s, 320 MB RAM
-  - rbtree and interval converted to cython:
-    8.4 μS, total 12 s, 134 MB RAM
-
-Layouts
--------
-Current/old design has specific layouts: RawData, PrepData, RawNotchedData.
-Let's get rid of this entirely and switch to simpler data types that are
-just collections and counts of a single type.  We'll still use strings
-to describe them, with format:
-
-    type_count
-
-where type is "uint16", "float32", or "float64", and count is an integer.
-
-nilmdb.layout.named() will parse these strings into the appropriate
-handlers.  For compatibility:
-
-    "RawData" == "uint16_6"
-    "RawNotchedData" == "uint16_9"
-    "PrepData" == "float32_8"

docs/Makefile

@@ -0,0 +1,9 @@
+ALL_DOCS = $(wildcard *.md)
+
+all: $(ALL_DOCS:.md=.html)
+
+%.html: %.md
+	pandoc -s $< > $@
+
+clean:
+	rm -f *.html

docs/TODO.md

@@ -0,0 +1,5 @@
+- Documentation
+
+- Machine-readable information in OverflowError, parser errors.
+  Maybe subclass `cherrypy.HTTPError` and override `set_response`
+  to add another JSON field?

docs/design.md

@@ -0,0 +1,391 @@
+Structure
+---------
+nilmdb.nilmdb is the NILM database interface.  A nilmdb.BulkData
+interface stores data in flat files, and a SQL database tracks
+metadata and ranges.
+
+Access to the nilmdb must be single-threaded.  This is handled with
+the nilmdb.serializer class.  In the future this could probably
+be turned into a per-path serialization.
+
+nilmdb.server is a HTTP server that provides an interface to talk,
+thorugh the serialization layer, to the nilmdb object.
+
+nilmdb.client is a HTTP client that connects to this.
+
+Sqlite performance
+------------------
+
+Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
+takes about 125msec.  sqlite3 will commit transactions at 3 times:
+
+1. explicit con.commit()
+
+2. between a series of DML commands and non-DML commands, e.g.
+   after a series of INSERT, SELECT, but before a CREATE TABLE or
+   PRAGMA.
+
+3. at the end of an explicit transaction, e.g. "with self.con as con:"
+
+To speed up testing, or if this transaction speed becomes an issue,
+the sync=False option to NilmDB will set PRAGMA synchronous=OFF.
+
+
+Inserting streams
+-----------------
+
+We need to send the contents of "data" as POST.  Do we need chunked
+transfer?
+
+- Don't know the size in advance, so we would need to use chunked if
+  we send the entire thing in one request.
+- But we shouldn't send one chunk per line, so we need to buffer some
+  anyway; why not just make new requests?
+- Consider the infinite-streaming case, we might want to send it
+  immediately?  Not really -- server still should do explicit inserts
+  of fixed-size chunks.
+- Even chunked encoding needs the size of each chunk beforehand, so
+  everything still gets buffered.  Just a tradeoff of buffer size.
+
+Before timestamps are added:
+
+- Raw data is about 440 kB/s    (9 channels)
+- Prep data is about 12.5 kB/s  (1 phase)
+- How do we know how much data to send?
+
+    - Remember that we can only do maybe 8-50 transactions per second on
+      the sqlite database.  So if one block of inserted data is one
+      transaction, we'd need the raw case to be around 64kB per request,
+      ideally more.
+    - Maybe use a range, based on how long it's taking to read the data
+        - If no more data, send it
+        - If data > 1 MB, send it
+    - If more than 10 seconds have elapsed, send it
+    - Should those numbers come from the server?
+
+Converting from ASCII to PyTables:
+
+- For each row getting added, we need to set attributes on a PyTables
+  Row object and call table.append().  This means that there isn't a
+  particularly efficient way of converting from ascii.
+- Could create a function like nilmdb.layout.Layout("foo".fillRow(asciiline)
+    - But this means we're doing parsing on the serialized side
+    - Let's keep parsing on the threaded server side so we can detect
+      errors better, and not block the serialized nilmdb for a slow
+      parsing process.
+- Client sends ASCII data
+- Server converts this ACSII data to a list of values
+    - Maybe:
+
+            # threaded side creates this object
+            parser = nilmdb.layout.Parser("layout_name")
+            # threaded side parses and fills it with data
+            parser.parse(textdata)
+            # serialized side pulls out rows
+            for n in xrange(parser.nrows):
+                parser.fill_row(rowinstance, n)
+                table.append()
+
+
+Inserting streams, inside nilmdb
+--------------------------------
+
+- First check that the new stream doesn't overlap.
+    - Get minimum timestamp, maximum timestamp from data parser.
+        - (extend parser to verify monotonicity and track extents)
+    - Get all intervals for this stream in the database
+    - See if new interval overlaps any existing ones
+        - If so, bail
+    - Question: should we cache intervals inside NilmDB?
+        - Assume database is fast for now, and always rebuild fom DB.
+        - Can add a caching layer later if we need to.
+    - `stream_get_ranges(path)` -> return IntervalSet?
+
+Speed
+-----
+
+- First approach was quadratic.  Adding four hours of data:
+
+        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/1/raw
+        real    24m31.093s
+        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/1/raw
+        real    43m44.528s
+        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-130002 /bpnilm/1/raw
+        real    93m29.713s
+        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-140003 /bpnilm/1/raw
+        real    166m53.007s
+
+- Disabling pytables indexing didn't help:
+
+        real    31m21.492s
+        real    52m51.963s
+        real    102m8.151s
+        real    176m12.469s
+
+- Server RAM usage is constant.
+
+- Speed problems were due to IntervalSet speed, of parsing intervals
+  from the database and adding the new one each time.
+
+    - First optimization is to cache result of `nilmdb:_get_intervals`,
+      which gives the best speedup.
+
+    - Also switched to internally using bxInterval from bx-python package.
+      Speed of `tests/test_interval:TestIntervalSpeed` is pretty decent
+      and seems to be growing logarithmically now.  About 85μs per insertion
+      for inserting 131k entries.
+
+    - Storing the interval data in SQL might be better, with a scheme like:
+      http://www.logarithmic.net/pfh/blog/01235197474
+
+- Next slowdown target is nilmdb.layout.Parser.parse().
+    - Rewrote parsers using cython and sscanf
+    - Stats (rev 10831), with `_add_interval` disabled
+
+        layout.pyx.Parser.parse:128        6303 sec, 262k calls
+         layout.pyx.parse:63               13913 sec, 5.1g calls
+        numpy:records.py.fromrecords:569   7410 sec, 262k calls
+
+- Probably OK for now.
+
+- After all updates, now takes about 8.5 minutes to insert an hour of
+  data, constant after adding 171 hours (4.9 billion data points)
+
+- Data set size: 98 gigs = 20 bytes per data point.
+  6 uint16 data + 1 uint32 timestamp = 16 bytes per point
+  So compression must be off -- will retry with compression forced on.
+
+IntervalSet speed
+-----------------
+- Initial implementation was pretty slow, even with binary search in
+  sorted list
+
+- Replaced with bxInterval; now takes about log n time for an insertion
+    - TestIntervalSpeed with range(17,18) and profiling
+        - 85 μs each
+        - 131072 calls to `__iadd__`
+        - 131072 to bx.insert_interval
+        - 131072 to bx.insert:395
+        - 2355835 to bx.insert:106  (18x as many?)
+
+- Tried blist too, worse than bxinterval.
+
+- Might be algorithmic improvements to be made in Interval.py,
+  like in `__and__`
+
+- Replaced again with rbtree.  Seems decent.  Numbers are time per
+  insert for 2**17 insertions, followed by total wall time and RAM
+  usage for running "make test" with `test_rbtree` and `test_interval`
+  with range(5,20):
+    - old values with bxinterval:
+      20.2 μS, total 20 s, 177 MB RAM
+    - rbtree, plain python:
+      97 μS, total 105 s, 846 MB RAM
+    - rbtree converted to cython:
+      26 μS, total 29 s, 320 MB RAM
+    - rbtree and interval converted to cython:
+      8.4 μS, total 12 s, 134 MB RAM
+
+- Would like to move Interval itself back to Python so other
+  non-cythonized code like client code can use it more easily.
+  Testing speed with just `test_interval` being tested, with
+  `range(5,22)`, using `/usr/bin/time -v python tests/runtests.py`,
+  times recorded for 2097152:
+    - 52ae397 (Interval in cython):
+	  12.6133 μs each, ratio 0.866533, total 47 sec, 399 MB RAM
+	- 9759dcf (Interval in python):
+	  21.2937 μs each, ratio 1.462870, total 83 sec, 1107 MB RAM
+  That's a huge difference!  Instead, will keep Interval and DBInterval
+  cythonized inside nilmdb, and just have an additional copy in
+  nilmdb.utils for clients to use.
+
+Layouts
+-------
+Current/old design has specific layouts: RawData, PrepData, RawNotchedData.
+Let's get rid of this entirely and switch to simpler data types that are
+just collections and counts of a single type.  We'll still use strings
+to describe them, with format:
+
+    type_count
+
+where type is "uint16", "float32", or "float64", and count is an integer.
+
+nilmdb.layout.named() will parse these strings into the appropriate
+handlers.  For compatibility:
+
+    "RawData" == "uint16_6"
+    "RawNotchedData" == "uint16_9"
+    "PrepData" == "float32_8"
+
+
+BulkData design
+---------------
+
+BulkData is a custom bulk data storage system that was written to
+replace PyTables.  The general structure is a `data` subdirectory in
+the main NilmDB directory.  Within `data`, paths are created for each
+created stream.  These locations are called tables.  For example,
+tables might be located at
+
+    nilmdb/data/newton/raw/
+    nilmdb/data/newton/prep/
+    nilmdb/data/cottage/raw/
+
+Each table contains:
+
+- An unchanging `_format` file (Python pickle format) that describes
+  parameters of how the data is broken up, like files per directory,
+  rows per file, and the binary data format
+
+- Hex named subdirectories `("%04x", although more than 65536 can exist)`
+
+- Hex named files within those subdirectories, like:
+
+        /nilmdb/data/newton/raw/000b/010a
+
+    The data format of these files is raw binary, interpreted by the
+    Python `struct` module according to the format string in the
+    `_format` file.
+
+- Same as above, with `.removed` suffix, is an optional file (Python
+  pickle format) containing a list of row numbers that have been
+  logically removed from the file.  If this range covers the entire
+  file, the entire file will be removed.
+
+- Note that the `bulkdata.nrows` variable is calculated once in
+  `BulkData.__init__()`, and only ever incremented during use.  Thus,
+  even if all data is removed, `nrows` can remain high.  However, if
+  the server is restarted, the newly calculated `nrows` may be lower
+  than in a previous run due to deleted data.  To be specific, this
+  sequence of events:
+
+    - insert data
+    - remove all data
+    - insert data
+
+    will result in having different row numbers in the database, and
+    differently numbered files on the filesystem, than the sequence:
+
+    - insert data
+    - remove all data
+    - restart server
+    - insert data
+
+    This is okay!  Everything should remain consistent both in the
+    `BulkData` and `NilmDB`.  Not attempting to readjust `nrows` during
+    deletion makes the code quite a bit simpler.
+
+- Similarly, data files are never truncated shorter.  Removing data
+  from the end of the file will not shorten it; it will only be
+  deleted when it has been fully filled and all of the data has been
+  subsequently removed.
+
+
+Rocket
+------
+
+Original design had the nilmdb.nilmdb thread (through bulkdata)
+convert from on-disk layout to a Python list, and then the
+nilmdb.server thread (from cherrypy) converts to ASCII.  For at least
+the extraction side of things, it's easy to pass the bulkdata a layout
+name instead, and have it convert directly from on-disk to ASCII
+format, because this conversion can then be shoved into a C module.
+This module, which provides a means for converting directly from
+on-disk format to ASCII or Python lists, is the "rocket" interface.
+Python is still used to manage the files and figure out where the
+data should go; rocket just puts binary data directly in or out of
+those files at specified locations.
+
+Before rocket, testing speed with uint16_6 data, with an end-to-end
+test (extracting data with nilmtool):
+
+- insert: 65 klines/sec
+- extract: 120 klines/sec
+
+After switching to the rocket design, but using the Python version
+(pyrocket):
+
+- insert: 57 klines/sec
+- extract: 120 klines/sec
+
+After switching to a C extension module (rocket.c)
+
+- insert: 74 klines/sec through insert.py; 99.6 klines/sec through nilmtool
+- extract: 335 klines/sec
+
+After client block updates (described below):
+
+- insert: 180 klines/sec through nilmtool (pre-timestamped)
+- extract: 390 klines/sec through nilmtool
+
+Using "insert --timestamp" or "extract --bare" cuts the speed in half.
+
+Blocks versus lines
+-------------------
+
+Generally want to avoid parsing the bulk of the data as lines if
+possible, and transfer things in bigger blocks at once.
+
+Current places where we use lines:
+
+- All data returned by `client.stream_extract`, since it comes from
+  `httpclient.get_gen`, which iterates over lines.  Not sure if this
+  should be changed, because a `nilmtool extract` is just about the
+  same speed as `curl -q .../stream/extract`!
+
+- `client.StreamInserter.insert_iter` and
+  `client.StreamInserter.insert_line`, which should probably get
+  replaced with block versions.  There's no real need to keep
+  updating the timestamp every time we get a new line of data.
+
+  - Finished.  Just a single insert() that takes any length string and
+    does very little processing until it's time to send it to the
+	server.
+
+Timestamps
+----------
+
+Timestamps are currently double-precision floats (64 bit).  Since the
+mantissa is 53-bit, this can only represent about 15-17 significant
+figures, and microsecond Unix timestamps like 1222333444.000111 are
+already 16 significant figures.  Rounding is therefore an issue;
+it's hard to sure that converting from ASCII, then back to ASCII,
+will always give the same result.
+
+Also, if the client provides a floating point value like 1.9999999999,
+we need to be careful that we don't store it as 1.9999999999 but later
+print it as 2.000000, because then round-trips change the data.
+
+Possible solutions:
+
+- When the client provides a floating point value to the server,
+  always round to the 6th decimal digit before verifying & storing.
+  Good for compatibility and simplicity.  But still might have rounding
+  issues, and clients will also need to round when doing their own
+  verification.  Having every piece of code need to know which digit
+  to round at is not ideal.
+
+- Always store int64 timestamps on the server, representing
+  microseconds since epoch.  int64 timestamps are used in all HTTP
+  parameters, in insert/extract ASCII strings, client API, commandline
+  raw timestamps, etc.  Pretty big change.
+
+  This is what we'll go with...
+
+  - Client programs that interpret the timestamps as doubles instead
+    of ints will remain accurate until 2^53 microseconds, or year
+    2255.
+
+  - On insert, maybe it's OK to send floating point microsecond values
+    (1234567890123456.0), just to cope with clients that want to print
+    everything as a double.  Server could try parsing as int64, and if
+    that fails, parse as double and truncate to int64.  However, this
+    wouldn't catch imprecise inputs like "1.23456789012e+15".  But
+    maybe that can just be ignored; it's likely to cause a
+    non-monotonic error at the client.
+
+  - Timestamps like 1234567890.123456 never show up anywhere, except
+    for interfacing to datetime_tz etc.  Command line "raw timestamps"
+    are always printed as int64 values, and a new format
+    "@1234567890123456" is added to the parser for specifying them
+    exactly.

docs/wsgi.md

@@ -0,0 +1,32 @@
+WSGI Application in Apache
+--------------------------
+
+Install `apache2` and `libapache2-mod-wsgi`
+
+We'll set up the database server at URL `http://myhost.com/nilmdb`.
+The database will be stored in `/home/nilm/db`, and the process will
+run as user `nilm`, group `nilm`.
+
+First, create a WSGI script `/home/nilm/nilmdb.wsgi` containing:
+
+    import nilmdb.server
+    application = nilmdb.server.wsgi_application("/home/nilm/db", "/nilmdb")
+
+The first parameter is the local filesystem path, and the second
+parameter is the path part of the URL.
+
+Then, set up Apache with a configuration like:
+
+    <VirtualHost>
+        WSGIScriptAlias /nilmdb /home/nilm/nilmdb.wsgi
+        WSGIApplicationGroup nilmdb-appgroup
+        WSGIProcessGroup nilmdb-procgroup
+        WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilm group=nilm
+
+        # Access control example:
+        <Location /nilmdb>
+            Order deny,allow
+            Deny from all
+            Allow from 1.2.3.4
+        </Location>
+    </VirtualHost>

extras/nilmtool-bash-completion.sh

@@ -0,0 +1,20 @@
+# To enable bash completion:
+#
+# 1. Ensure python-argcomplete is installed:
+#       pip install argcomplete
+# 2. Source this file:
+#       . nilmtool-bash-completion.sh
+
+_nilmtool_argcomplete() {
+    local IFS=$(printf "\013")
+    COMPREPLY=( $(IFS="$IFS" \
+                  COMP_LINE="$COMP_LINE" \
+	          COMP_WORDBREAKS="$COMP_WORDBREAKS" \
+                  COMP_POINT="$COMP_POINT" \
+                  _ARGCOMPLETE=1 \
+                  "$1" 8>&1 9>&2 1>/dev/null 2>/dev/null) )
+    if [[ $? != 0 ]]; then
+        unset COMPREPLY
+    fi
+}
+complete -o nospace -F _nilmtool_argcomplete nilmtool

nilmdb/__init__.py

@@ -1,12 +1,10 @@
 """Main NilmDB import"""
 
-from .nilmdb import NilmDB
-from .server import Server
-from .client import Client
-
-import pyximport; pyximport.install()
-import layout
-import interval
-
-import cmdline
+# These aren't imported automatically, because loading the server
+# stuff isn't always necessary.
+#from nilmdb.server import NilmDB, Server
+#from nilmdb.client import Client
 
+from nilmdb._version import get_versions
+__version__ = get_versions()['version']
+del get_versions

nilmdb/_version.py

@@ -0,0 +1,197 @@
+
+IN_LONG_VERSION_PY = True
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by githubs download-from-tag
+# feature). Distribution tarballs (build by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
+
+# This file is released into the public domain. Generated by
+# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+
+# these strings will be replaced by git during git-archive
+git_refnames = "$Format:%d$"
+git_full = "$Format:%H$"
+
+
+import subprocess
+import sys
+
+def run_command(args, cwd=None, verbose=False):
+    try:
+        # remember shell=False, so use git.cmd on windows, not just git
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
+    except EnvironmentError:
+        e = sys.exc_info()[1]
+        if verbose:
+            print("unable to run %s" % args[0])
+            print(e)
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version >= '3':
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % args[0])
+        return None
+    return stdout
+
+
+import sys
+import re
+import os.path
+
+def get_expanded_variables(versionfile_source):
+    # the code embedded in _version.py can just fetch the value of these
+    # variables. When used from setup.py, we don't want to import
+    # _version.py, so we do it with a regexp instead. This function is not
+    # used from _version.py.
+    variables = {}
+    try:
+        for line in open(versionfile_source,"r").readlines():
+            if line.strip().startswith("git_refnames ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["refnames"] = mo.group(1)
+            if line.strip().startswith("git_full ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    variables["full"] = mo.group(1)
+    except EnvironmentError:
+        pass
+    return variables
+
+def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
+    refnames = variables["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("variables are unexpanded, not using")
+        return {} # unexpanded, so not in an unpacked git-archive tarball
+    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    for ref in list(refs):
+        if not re.search(r'\d', ref):
+            if verbose:
+                print("discarding '%s', no digits" % ref)
+            refs.discard(ref)
+            # Assume all version tags have a digit. git's %d expansion
+            # behaves like git log --decorate=short and strips out the
+            # refs/heads/ and refs/tags/ prefixes that would let us
+            # distinguish between branches and tags. By ignoring refnames
+            # without digits, we filter out many common branch names like
+            # "release" and "stabilization", as well as "HEAD" and "master".
+    if verbose:
+        print("remaining refs: %s" % ",".join(sorted(refs)))
+    for ref in sorted(refs):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix):]
+            if verbose:
+                print("picking %s" % r)
+            return { "version": r,
+                     "full": variables["full"].strip() }
+    # no suitable tags, so we use the full revision id
+    if verbose:
+        print("no suitable tags, using full revision id")
+    return { "version": variables["full"].strip(),
+             "full": variables["full"].strip() }
+
+def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
+    # this runs 'git' from the root of the source tree. That either means
+    # someone ran a setup.py command (and this code is in versioneer.py, so
+    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
+    # the source tree), or someone ran a project-specific entry point (and
+    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
+    # containing directory is somewhere deeper in the source tree). This only
+    # gets called if the git-archive 'subst' variables were *not* expanded,
+    # and _version.py hasn't already been rewritten with a short version
+    # string, meaning we're inside a checked out source tree.
+
+    try:
+        here = os.path.abspath(__file__)
+    except NameError:
+        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
+        return {} # not always correct
+
+    # versionfile_source is the relative path from the top of the source tree
+    # (where the .git directory might live) to this file. Invert this to find
+    # the root from __file__.
+    root = here
+    if IN_LONG_VERSION_PY:
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        root = os.path.dirname(here)
+    if not os.path.exists(os.path.join(root, ".git")):
+        if verbose:
+            print("no .git in %s" % root)
+        return {}
+
+    GIT = "git"
+    if sys.platform == "win32":
+        GIT = "git.cmd"
+    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
+                         cwd=root)
+    if stdout is None:
+        return {}
+    if not stdout.startswith(tag_prefix):
+        if verbose:
+            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
+        return {}
+    tag = stdout[len(tag_prefix):]
+    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
+    if stdout is None:
+        return {}
+    full = stdout.strip()
+    if tag.endswith("-dirty"):
+        full += "-dirty"
+    return {"version": tag, "full": full}
+
+
+def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
+    if IN_LONG_VERSION_PY:
+        # We're running from _version.py. If it's from a source tree
+        # (execute-in-place), we can work upwards to find the root of the
+        # tree, and then check the parent directory for a version string. If
+        # it's in an installed application, there's no hope.
+        try:
+            here = os.path.abspath(__file__)
+        except NameError:
+            # py2exe/bbfreeze/non-CPython don't have __file__
+            return {} # without __file__, we have no hope
+        # versionfile_source is the relative path from the top of the source
+        # tree to _version.py. Invert this to find the root from __file__.
+        root = here
+        for i in range(len(versionfile_source.split("/"))):
+            root = os.path.dirname(root)
+    else:
+        # we're running from versioneer.py, which means we're running from
+        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
+        here = os.path.abspath(sys.argv[0])
+        root = os.path.dirname(here)
+
+    # Source tarballs conventionally unpack into a directory that includes
+    # both the project name and a version string.
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
+                  (root, dirname, parentdir_prefix))
+        return None
+    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+
+tag_prefix = "nilmdb-"
+parentdir_prefix = "nilmdb-"
+versionfile_source = "nilmdb/_version.py"
+
+def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
+    variables = { "refnames": git_refnames, "full": git_full }
+    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
+    if not ver:
+        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
+    if not ver:
+        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
+                                      verbose)
+    if not ver:
+        ver = default
+    return ver
+

nilmdb/bulkdata.py

@@ -1,310 +0,0 @@
-# Fixed record size bulk data storage
-
-from __future__ import absolute_import
-from __future__ import division
-import nilmdb
-from nilmdb.utils.printf import *
-
-import os
-import sys
-import cPickle as pickle
-import struct
-import fnmatch
-import mmap
-
-# Up to 256 open file descriptors at any given time
-table_cache_size = 16
-fd_cache_size = 16
-
-@nilmdb.utils.must_close()
-class BulkData(object):
-    def __init__(self, basepath):
-        self.basepath = basepath
-        self.root = os.path.join(self.basepath, "data")
-
-        # Make root path
-        if not os.path.isdir(self.root):
-            os.mkdir(self.root)
-
-    def close(self):
-        self.getnode.cache_remove_all()
-
-    def _encode_filename(self, path):
-        # Encode all paths to UTF-8, regardless of sys.getfilesystemencoding(),
-        # because we want to be able to represent all code points and the user
-        # will never be directly exposed to filenames.  We can then do path
-        # manipulations on the UTF-8 directly.
-        if isinstance(path, unicode):
-            return path.encode('utf-8')
-        return path
-
-    def create(self, unicodepath, layout_name):
-        """
-        unicodepath: path to the data (e.g. u'/newton/prep').
-        Paths must contain at least two elements, e.g.:
-           /newton/prep
-           /newton/raw
-           /newton/upstairs/prep
-           /newton/upstairs/raw
-
-        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
-        """
-        path = self._encode_filename(unicodepath)
-
-        if path[0] != '/':
-            raise ValueError("paths must start with /")
-        [ group, node ] = path.rsplit("/", 1)
-        if group == '':
-            raise ValueError("invalid path")
-
-        # Get layout, and build format string for struct module
-        try:
-            layout = nilmdb.layout.get_named(layout_name)
-            struct_fmt = '<d'  # Little endian, double timestamp
-            struct_mapping = {
-                "int8": 'b',
-                "uint8": 'B',
-                "int16": 'h',
-                "uint16": 'H',
-                "int32": 'i',
-                "uint32": 'I',
-                "int64": 'q',
-                "uint64": 'Q',
-                "float32": 'f',
-                "float64": 'd',
-                }
-            for n in range(layout.count):
-                struct_fmt += struct_mapping[layout.datatype]
-        except KeyError:
-            raise ValueError("no such layout, or bad data types")
-
-        # Create the table.  Note that we make a distinction here
-        # between NilmDB paths (always Unix style, split apart
-        # manually) and OS paths (built up with os.path.join)
-        try:
-            # Make directories leading up to this one
-            elements = path.lstrip('/').split('/')
-            for i in range(len(elements)):
-                ospath = os.path.join(self.root, *elements[0:i])
-                if Table.exists(ospath):
-                    raise ValueError("path is subdir of existing node")
-                if not os.path.isdir(ospath):
-                    os.mkdir(ospath)
-
-            # Make the final dir
-            ospath = os.path.join(self.root, *elements)
-            if os.path.isdir(ospath):
-                raise ValueError("subdirs of this path already exist")
-            os.mkdir(ospath)
-
-            # Write format string to file
-            Table.create(ospath, struct_fmt)
-        except OSError as e:
-            raise ValueError("error creating table at that path: " + e.strerror)
-
-        # Open and cache it
-        self.getnode(unicodepath)
-
-        # Success
-        return
-
-    def destroy(self, unicodepath):
-        """Fully remove all data at a particular path.  No way to undo
-        it!  The group/path structure is removed, too."""
-        path = self._encode_filename(unicodepath)
-
-        # Get OS path
-        elements = path.lstrip('/').split('/')
-        ospath = os.path.join(self.root, *elements)
-
-        # Remove Table object from cache
-        self.getnode.cache_remove(self, ospath)
-
-        # Remove the contents of the target directory
-        if not os.path.isfile(os.path.join(ospath, "format")):
-            raise ValueError("nothing at that path")
-        for file in os.listdir(ospath):
-            os.remove(os.path.join(ospath, file))
-
-        # Remove empty parent directories
-        for i in reversed(range(len(elements))):
-            ospath = os.path.join(self.root, *elements[0:i+1])
-            try:
-                os.rmdir(ospath)
-            except OSError:
-                break
-
-    # Cache open tables
-    @nilmdb.utils.lru_cache(size = table_cache_size,
-                            onremove = lambda x: x.close())
-    def getnode(self, unicodepath):
-        """Return a Table object corresponding to the given database
-        path, which must exist."""
-        path = self._encode_filename(unicodepath)
-        elements = path.lstrip('/').split('/')
-        ospath = os.path.join(self.root, *elements)
-        return Table(ospath)
-
-@nilmdb.utils.must_close()
-class Table(object):
-    """Tools to help access a single table (data at a specific OS path)"""
-
-    # Class methods, to help keep format details in this class.
-    @classmethod
-    def exists(cls, root):
-        """Return True if a table appears to exist at this OS path"""
-        return os.path.isfile(os.path.join(root, "format"))
-
-    @classmethod
-    def create(cls, root, struct_fmt):
-        """Initialize a table at the given OS path.
-        'struct_fmt' is a Struct module format description"""
-        format = { "rows_per_file": 4 * 1024 * 1024,
-                   "struct_fmt": struct_fmt }
-        with open(os.path.join(root, "format"), "wb") as f:
-            pickle.dump(format, f, 2)
-
-    # Normal methods
-    def __init__(self, root):
-        """'root' is the full OS path to the directory of this table"""
-        self.root = root
-
-        # Load the format and build packer
-        with open(self._fullpath("format"), "rb") as f:
-            format = pickle.load(f)
-        self.rows_per_file = format["rows_per_file"]
-        self.packer = struct.Struct(format["struct_fmt"])
-        self.file_size = self.packer.size * self.rows_per_file
-
-        # Find nrows by locating the lexicographically last filename
-        # and using its size.
-        pattern = '[0-9a-f]' * 8
-        allfiles = fnmatch.filter(os.listdir(self.root), pattern)
-        if allfiles:
-            filename = max(allfiles)
-            offset = os.path.getsize(self._fullpath(filename))
-            self.nrows = self._row_from_fnoffset(filename, offset)
-        else:
-            self.nrows = 0
-
-    def close(self):
-        self.mmap_open.cache_remove_all()
-
-    # Internal helpers
-    def _fullpath(self, filename):
-        return os.path.join(self.root, filename)
-
-    def _fnoffset_from_row(self, row):
-        """Return a (filename, offset, count) tuple:
-
-        filename: the filename that contains the specified row
-          offset: byte offset of the specified row within the file
-           count: number of rows (starting at offste) that fit in the file
-        """
-        filenum = row // self.rows_per_file
-        filename = sprintf("%08x", filenum)
-        offset = (row % self.rows_per_file) * self.packer.size
-        count = self.rows_per_file - (row % self.rows_per_file)
-        return (filename, offset, count)
-
-    def _row_from_fnoffset(self, filename, offset):
-        """Return the row number that corresponds to the given
-        filename and byte-offset within that file."""
-        filenum = int(filename, 16)
-        if (offset % self.packer.size) != 0:
-            raise ValueError("file offset is not a multiple of data size")
-        row = (filenum * self.rows_per_file) + (offset // self.packer.size)
-        return row
-
-    # Cache open files
-    @nilmdb.utils.lru_cache(size = fd_cache_size,
-                            onremove = lambda x: x.close())
-    def mmap_open(self, file, newsize = None):
-        """Open and map a given filename (relative to self.root).
-        Will be automatically closed when evicted from the cache.
-
-        If 'newsize' is provided, the file is truncated to the given
-        size before the mapping is returned.  (Note that the LRU cache
-        on this function means the truncate will only happen if the
-        object isn't already cached; mmap.resize should be used too)"""
-        f = open(os.path.join(self.root, file), "a+", 0)
-        if newsize is not None:
-            # mmap can't map a zero-length file, so this allows the
-            # caller to set the filesize between file creation and
-            # mmap.
-            f.truncate(newsize)
-        mm = mmap.mmap(f.fileno(), 0)
-        return mm
-
-    def append(self, data):
-        """Append the data and flush it to disk.
-        data is a nested Python list [[row],[row],[...]]"""
-        remaining = len(data)
-        dataiter = iter(data)
-        while remaining:
-            # See how many rows we can fit into the current file, and open it
-            (filename, offset, count) = self._fnoffset_from_row(self.nrows)
-            if count > remaining:
-                count = remaining
-            newsize = offset + count * self.packer.size
-            mm = self.mmap_open(filename, newsize)
-            mm.seek(offset)
-
-            # Extend the file to the target length.  We specified
-            # newsize when opening, but that may have been ignored if
-            # the mmap_open returned a cached object.
-            mm.resize(newsize)
-
-            # Write the data
-            for i in xrange(count):
-                row = dataiter.next()
-                mm.write(self.packer.pack(*row))
-            remaining -= count
-            self.nrows += count
-
-    def __getitem__(self, key):
-        """Extract data and return it.  Supports simple indexing
-        (table[n]) and range slices (table[n:m]).  Returns a nested
-        Python list [[row],[row],[...]]"""
-
-        # Handle simple slices
-        if isinstance(key, slice):
-            # Fall back to brute force if the slice isn't simple
-            if ((key.step is not None and key.step != 1) or
-                key.start is None or
-                key.stop is None or
-                key.start >= key.stop or
-                key.start < 0 or
-                key.stop > self.nrows):
-                return [ self[x] for x in xrange(*key.indices(self.nrows)) ]
-
-            ret = []
-            row = key.start
-            remaining = key.stop - key.start
-            while remaining:
-                (filename, offset, count) = self._fnoffset_from_row(row)
-                if count > remaining:
-                    count = remaining
-                mm = self.mmap_open(filename)
-                for i in xrange(count):
-                    ret.append(list(self.packer.unpack_from(mm, offset)))
-                    offset += self.packer.size
-                remaining -= count
-                row += count
-            return ret
-
-        # Handle single points
-        if key < 0 or key >= self.nrows:
-            raise IndexError("Index out of range")
-        (filename, offset, count) = self._fnoffset_from_row(key)
-        mm = self.mmap_open(filename)
-        # unpack_from ignores the mmap object's current seek position
-        return self.packer.unpack_from(mm, offset)
-
-class TimestampOnlyTable(object):
-    """Helper that lets us pass a Tables object into bisect, by
-    returning only the timestamp when a particular row is requested."""
-    def __init__(self, table):
-        self.table = table
-    def __getitem__(self, index):
-        return self.table[index][0]

nilmdb/client.py

@@ -1,208 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Class for performing HTTP client requests via libcurl"""
-
-from __future__ import absolute_import
-from nilmdb.utils.printf import *
-
-import time
-import sys
-import re
-import os
-import simplejson as json
-import itertools
-
-import nilmdb.httpclient
-
-# Other functions expect to see these in the nilmdb.client namespace
-from nilmdb.httpclient import ClientError, ServerError, Error
-
-version = "1.0"
-
-def float_to_string(f):
-    # Use repr to maintain full precision in the string output.
-    return repr(float(f))
-
-class Client(object):
-    """Main client interface to the Nilm database."""
-
-    client_version = version
-
-    def __init__(self, url):
-        self.http = nilmdb.httpclient.HTTPClient(url)
-
-    def _json_param(self, data):
-        """Return compact json-encoded version of parameter"""
-        return json.dumps(data, separators=(',',':'))
-
-    def close(self):
-        self.http.close()
-
-    def geturl(self):
-        """Return the URL we're using"""
-        return self.http.baseurl
-
-    def version(self):
-        """Return server version"""
-        return self.http.get("version")
-
-    def dbpath(self):
-        """Return server database path"""
-        return self.http.get("dbpath")
-
-    def dbsize(self):
-        """Return server database size as human readable string"""
-        return self.http.get("dbsize")
-
-    def stream_list(self, path = None, layout = None):
-        params = {}
-        if path is not None:
-            params["path"] = path
-        if layout is not None:
-            params["layout"] = layout
-        return self.http.get("stream/list", params)
-
-    def stream_get_metadata(self, path, keys = None):
-        params = { "path": path }
-        if keys is not None:
-            params["key"] = keys
-        return self.http.get("stream/get_metadata", params)
-
-    def stream_set_metadata(self, path, data):
-        """Set stream metadata from a dictionary, replacing all existing
-        metadata."""
-        params = {
-            "path": path,
-            "data": self._json_param(data)
-            }
-        return self.http.get("stream/set_metadata", params)
-
-    def stream_update_metadata(self, path, data):
-        """Update stream metadata from a dictionary"""
-        params = {
-            "path": path,
-            "data": self._json_param(data)
-            }
-        return self.http.get("stream/update_metadata", params)
-
-    def stream_create(self, path, layout):
-        """Create a new stream"""
-        params = { "path": path,
-                   "layout" : layout }
-        return self.http.get("stream/create", params)
-
-    def stream_destroy(self, path):
-        """Delete stream and its contents"""
-        params = { "path": path }
-        return self.http.get("stream/destroy", params)
-
-    def stream_insert(self, path, data, start = None, end = None):
-        """Insert data into a stream.  data should be a file-like object
-        that provides ASCII data that matches the database layout for path.
-
-        start and end are the starting and ending timestamp of this
-        stream; all timestamps t in the data must satisfy 'start <= t
-        < end'.  If left unspecified, 'start' is the timestamp of the
-        first line of data, and 'end' is the timestamp on the last line
-        of data, plus a small delta of 1μs.
-        """
-        params = { "path": path }
-
-        # See design.md for a discussion of how much data to send.
-        # These are soft limits -- actual data might be rounded up.
-        max_data = 1048576
-        max_time = 30
-        end_epsilon = 1e-6
-
-        def pairwise(iterable):
-            "s -> (s0,s1), (s1,s2), ..., (sn,None)"
-            a, b = itertools.tee(iterable)
-            next(b, None)
-            return itertools.izip_longest(a, b)
-
-        def extract_timestamp(line):
-            return float(line.split()[0])
-
-        def sendit():
-            # If we have more data after this, use the timestamp of
-            # the next line as the end.  Otherwise, use the given
-            # overall end time, or add end_epsilon to the last data
-            # point.
-            if nextline:
-                block_end = extract_timestamp(nextline)
-                if end and block_end > end:
-                    # This is unexpected, but we'll defer to the server
-                    # to return an error in this case.
-                    block_end = end
-            elif end:
-                block_end = end
-            else:
-                block_end = extract_timestamp(line) + end_epsilon
-
-            # Send it
-            params["start"] = float_to_string(block_start)
-            params["end"] = float_to_string(block_end)
-            return self.http.put("stream/insert", block_data, params)
-
-        clock_start = time.time()
-        block_data = ""
-        block_start = start
-        result = None
-        for (line, nextline) in pairwise(data):
-            # If we don't have a starting time, extract it from the first line
-            if block_start is None:
-                block_start = extract_timestamp(line)
-
-            clock_elapsed = time.time() - clock_start
-            block_data += line
-
-            # If we have enough data, or enough time has elapsed,
-            # send this block to the server, and empty things out
-            # for the next block.
-            if (len(block_data) > max_data) or (clock_elapsed > max_time):
-                result = sendit()
-                block_start = None
-                block_data = ""
-                clock_start = time.time()
-
-        # One last block?
-        if len(block_data):
-            result = sendit()
-
-        # Return the most recent JSON result we got back, or None if
-        # we didn't make any requests.
-        return result
-
-    def stream_intervals(self, path, start = None, end = None):
-        """
-        Return a generator that yields each stream interval.
-        """
-        params = {
-            "path": path
-        }
-        if start is not None:
-            params["start"] = float_to_string(start)
-        if end is not None:
-            params["end"] = float_to_string(end)
-        return self.http.get_gen("stream/intervals", params, retjson = True)
-
-    def stream_extract(self, path, start = None, end = None, count = False):
-        """
-        Extract data from a stream.  Returns a generator that yields
-        lines of ASCII-formatted data that matches the database
-        layout for the given path.
-
-        Specify count=True to just get a count of values rather than
-        the actual data.
-        """
-        params = {
-            "path": path,
-        }
-        if start is not None:
-            params["start"] = float_to_string(start)
-        if end is not None:
-            params["end"] = float_to_string(end)
-        if count:
-            params["count"] = 1
-
-        return self.http.get_gen("stream/extract", params, retjson = False)

nilmdb/client/__init__.py

@@ -0,0 +1,4 @@
+"""nilmdb.client"""
+
+from nilmdb.client.client import Client
+from nilmdb.client.errors import ClientError, ServerError, Error

nilmdb/client/client.py

@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+
+"""Class for performing HTTP client requests via libcurl"""
+
+import nilmdb.utils
+import nilmdb.client.httpclient
+from nilmdb.client.errors import ClientError
+
+import time
+import simplejson as json
+import contextlib
+
+from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
+
+def extract_timestamp(line):
+    """Extract just the timestamp from a line of data text"""
+    return string_to_timestamp(line.split()[0])
+
+class Client(object):
+    """Main client interface to the Nilm database."""
+
+    def __init__(self, url, post_json = False):
+        """Initialize client with given URL.  If post_json is true,
+        POST requests are sent with Content-Type 'application/json'
+        instead of the default 'x-www-form-urlencoded'."""
+        self.http = nilmdb.client.httpclient.HTTPClient(url, post_json)
+        self.post_json = post_json
+
+    # __enter__/__exit__ allow this class to be a context manager
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+    def _json_post_param(self, data):
+        """Return compact json-encoded version of parameter"""
+        if self.post_json:
+            # If we're posting as JSON, we don't need to encode it further here
+            return data
+        return json.dumps(data, separators=(',',':'))
+
+    def close(self):
+        """Close the connection; safe to call multiple times"""
+        self.http.close()
+
+    def geturl(self):
+        """Return the URL we're using"""
+        return self.http.baseurl
+
+    def version(self):
+        """Return server version"""
+        return self.http.get("version")
+
+    def dbinfo(self):
+        """Return server database info (path, size, free space)
+        as a dictionary."""
+        return self.http.get("dbinfo")
+
+    def stream_list(self, path = None, layout = None, extended = False):
+        params = {}
+        if path is not None:
+            params["path"] = path
+        if layout is not None:
+            params["layout"] = layout
+        if extended:
+            params["extended"] = 1
+        return self.http.get("stream/list", params)
+
+    def stream_get_metadata(self, path, keys = None):
+        params = { "path": path }
+        if keys is not None:
+            params["key"] = keys
+        return self.http.get("stream/get_metadata", params)
+
+    def stream_set_metadata(self, path, data):
+        """Set stream metadata from a dictionary, replacing all existing
+        metadata."""
+        params = {
+            "path": path,
+            "data": self._json_post_param(data)
+            }
+        return self.http.post("stream/set_metadata", params)
+
+    def stream_update_metadata(self, path, data):
+        """Update stream metadata from a dictionary"""
+        params = {
+            "path": path,
+            "data": self._json_post_param(data)
+            }
+        return self.http.post("stream/update_metadata", params)
+
+    def stream_create(self, path, layout):
+        """Create a new stream"""
+        params = { "path": path,
+                   "layout" : layout }
+        return self.http.post("stream/create", params)
+
+    def stream_destroy(self, path):
+        """Delete stream.  Fails if any data is still present."""
+        params = { "path": path }
+        return self.http.post("stream/destroy", params)
+
+    def stream_rename(self, oldpath, newpath):
+        """Rename a stream."""
+        params = { "oldpath": oldpath,
+                   "newpath": newpath }
+        return self.http.post("stream/rename", params)
+
+    def stream_remove(self, path, start = None, end = None):
+        """Remove data from the specified time range"""
+        params = {
+            "path": path
+        }
+        if start is not None:
+            params["start"] = timestamp_to_string(start)
+        if end is not None:
+            params["end"] = timestamp_to_string(end)
+        return self.http.post("stream/remove", params)
+
+    @contextlib.contextmanager
+    def stream_insert_context(self, path, start = None, end = None):
+        """Return a context manager that allows data to be efficiently
+        inserted into a stream in a piecewise manner.  Data is be provided
+        as single lines, and is aggregated and sent to the server in larger
+        chunks as necessary.  Data lines must match the database layout for
+        the given path, and end with a newline.
+
+        Example:
+          with client.stream_insert_context('/path', start, end) as ctx:
+            ctx.insert('1234567890.0 1 2 3 4\\n')
+            ctx.insert('1234567891.0 1 2 3 4\\n')
+
+        For more details, see help for nilmdb.client.client.StreamInserter
+
+        This may make multiple requests to the server, if the data is
+        large enough or enough time has passed between insertions.
+        """
+        ctx = StreamInserter(self.http, path, start, end)
+        yield ctx
+        ctx.finalize()
+
+    def stream_insert(self, path, data, start = None, end = None):
+        """Insert rows of data into a stream.  data should be a string
+        or iterable that provides ASCII data that matches the database
+        layout for path.  See stream_insert_context for details on the
+        'start' and 'end' parameters."""
+        with self.stream_insert_context(path, start, end) as ctx:
+            if isinstance(data, basestring):
+                ctx.insert(data)
+            else:
+                for chunk in data:
+                    ctx.insert(chunk)
+        return ctx.last_response
+
+    def stream_intervals(self, path, start = None, end = None, diffpath = None):
+        """
+        Return a generator that yields each stream interval.
+
+        If diffpath is not None, yields only interval ranges that are
+        present in 'path' but not in 'diffpath'.
+        """
+        params = {
+            "path": path
+        }
+        if diffpath is not None:
+            params["diffpath"] = diffpath
+        if start is not None:
+            params["start"] = timestamp_to_string(start)
+        if end is not None:
+            params["end"] = timestamp_to_string(end)
+        return self.http.get_gen("stream/intervals", params)
+
+    def stream_extract(self, path, start = None, end = None,
+                       count = False, markup = False):
+        """
+        Extract data from a stream.  Returns a generator that yields
+        lines of ASCII-formatted data that matches the database
+        layout for the given path.
+
+        Specify count = True to return a count of matching data points
+        rather than the actual data.  The output format is unchanged.
+
+        Specify markup = True to include comments in the returned data
+        that indicate interval starts and ends.
+        """
+        params = {
+            "path": path,
+        }
+        if start is not None:
+            params["start"] = timestamp_to_string(start)
+        if end is not None:
+            params["end"] = timestamp_to_string(end)
+        if count:
+            params["count"] = 1
+        if markup:
+            params["markup"] = 1
+        return self.http.get_gen("stream/extract", params)
+
+    def stream_count(self, path, start = None, end = None):
+        """
+        Return the number of rows of data in the stream that satisfy
+        the given timestamps.
+        """
+        counts = list(self.stream_extract(path, start, end, count = True))
+        return int(counts[0])
+
+class StreamInserter(object):
+    """Object returned by stream_insert_context() that manages
+    the insertion of rows of data into a particular path.
+
+    The basic data flow is that we are filling a contiguous interval
+    on the server, with no gaps, that extends from timestamp 'start'
+    to timestamp 'end'.  Data timestamps satisfy 'start <= t < end'.
+
+    Data is provided to .insert() as ASCII formatted data separated by
+    newlines.  The chunks of data passed to .insert() do not need to
+    match up with the newlines; less or more than one line can be passed.
+
+    1. The first inserted line begins a new interval that starts at
+    'start'.  If 'start' is not given, it is deduced from the first
+    line's timestamp.
+
+    2. Subsequent lines go into the same contiguous interval.  As lines
+    are inserted, this routine may make multiple insertion requests to
+    the server, but will structure the timestamps to leave no gaps.
+
+    3. The current contiguous interval can be completed by manually
+    calling .finalize(), which the context manager will also do
+    automatically.  This will send any remaining data to the server,
+    using the 'end' timestamp to end the interval.  If no 'end'
+    was provided, it is deduced from the last timestamp seen,
+    plus a small delta.
+
+    After a .finalize(), inserting new data goes back to step 1.
+
+    .update_start() can be called before step 1 to change the start
+    time for the interval.  .update_end() can be called before step 3
+    to change the end time for the interval.
+    """
+
+    # See design.md for a discussion of how much data to send.  This
+    # is a soft limit -- we might send up to twice as much or so
+    _max_data = 2 * 1024 * 1024
+    _max_data_after_send = 64 * 1024
+
+    def __init__(self, http, path, start = None, end = None):
+        """'http' is the httpclient object.  'path' is the database
+        path to insert to.  'start' and 'end' are used for the first
+        contiguous interval."""
+        self.last_response = None
+
+        self._http = http
+        self._path = path
+
+        # Start and end for the overall contiguous interval we're
+        # filling
+        self._interval_start = start
+        self._interval_end = end
+
+        # Current data we're building up to send.  Each string
+        # goes into the array, and gets joined all at once.
+        self._block_data = []
+        self._block_len = 0
+
+    def insert(self, data):
+        """Insert a chunk of ASCII formatted data in string form.  The
+        overall data must consist of lines terminated by '\\n'."""
+        length = len(data)
+        maxdata = self._max_data
+
+        if length > maxdata:
+            # This could make our buffer more than twice what we
+            # wanted to send, so split it up.  This is a bit
+            # inefficient, but the user really shouldn't be providing
+            # this much data at once.
+            for cut in range(0, length, maxdata):
+                self.insert(data[cut:(cut + maxdata)])
+            return
+
+        # Append this string to our list
+        self._block_data.append(data)
+        self._block_len += length
+
+        # Send the block once we have enough data
+        if self._block_len >= maxdata:
+            self._send_block(final = False)
+            if self._block_len >= self._max_data_after_send: # pragma: no cover
+                raise ValueError("too much data left over after trying"
+                                 " to send intermediate block; is it"
+                                 " missing newlines or malformed?")
+
+    def update_start(self, start):
+        """Update the start time for the next contiguous interval.
+        Call this before starting to insert data for a new interval,
+        for example, after .finalize()"""
+        self._interval_start = start
+
+    def update_end(self, end):
+        """Update the end time for the current contiguous interval.
+        Call this before .finalize()"""
+        self._interval_end = end
+
+    def finalize(self):
+        """Stop filling the current contiguous interval.
+        All outstanding data will be sent, and the interval end
+        time of the interval will be taken from the 'end' argument
+        used when initializing this class, or the most recent
+        value passed to update_end(), or the last timestamp plus
+        a small epsilon value if no other endpoint was provided.
+
+        If more data is inserted after a finalize(), it will become
+        part of a new interval and there may be a gap left in-between."""
+        self._send_block(final = True)
+
+    def send(self):
+        """Send any data that we might have buffered up.  Does not affect
+        any other treatment of timestamps or endpoints."""
+        self._send_block(final = False)
+
+    def _get_first_noncomment(self, block):
+        """Return the (start, end) indices of the first full line in
+        block that isn't a comment, or raise IndexError if
+        there isn't one."""
+        start = 0
+        while True:
+            end = block.find('\n', start)
+            if end < 0:
+                raise IndexError
+            if block[start] != '#':
+                return (start, (end + 1))
+            start = end + 1
+
+    def _get_last_noncomment(self, block):
+        """Return the (start, end) indices of the last full line in
+        block[:length] that isn't a comment, or raise IndexError if
+        there isn't one."""
+        end = block.rfind('\n')
+        if end <= 0:
+            raise IndexError
+        while True:
+            start = block.rfind('\n', 0, end)
+            if block[start + 1] != '#':
+                return ((start + 1), end)
+            if start == -1:
+                raise IndexError
+            end = start
+
+    def _send_block(self, final = False):
+        """Send data currently in the block.  The data sent will
+        consist of full lines only, so some might be left over."""
+        # Build the full string to send
+        block = "".join(self._block_data)
+
+        start_ts = self._interval_start
+        if start_ts is None:
+            # Pull start from the first line
+            try:
+                (spos, epos) = self._get_first_noncomment(block)
+                start_ts = extract_timestamp(block[spos:epos])
+            except (ValueError, IndexError):
+                pass # no timestamp is OK, if we have no data
+
+        if final:
+            # For a final block, it must end in a newline, and the
+            # ending timestamp is either the user-provided end,
+            # or the timestamp of the last line plus epsilon.
+            end_ts = self._interval_end
+            try:
+                if block[-1] != '\n':
+                    raise ValueError("final block didn't end with a newline")
+                if end_ts is None:
+                    (spos, epos) = self._get_last_noncomment(block)
+                    end_ts = extract_timestamp(block[spos:epos])
+                    end_ts += nilmdb.utils.time.epsilon
+            except (ValueError, IndexError):
+                pass # no timestamp is OK, if we have no data
+            self._block_data = []
+            self._block_len = 0
+
+            # Next block is completely fresh
+            self._interval_start = None
+            self._interval_end = None
+        else:
+            # An intermediate block, e.g. "line1\nline2\nline3\nline4"
+            # We need to save "line3\nline4" for the next block, and
+            # use the timestamp from "line3" as the ending timestamp
+            # for this one.
+            try:
+                (spos, epos) = self._get_last_noncomment(block)
+                end_ts = extract_timestamp(block[spos:epos])
+            except (ValueError, IndexError):
+                # If we found no timestamp, give up; we could send this
+                # block later when we have more data.
+                return
+            if spos == 0:
+                # Not enough data to send an intermediate block
+                return
+            if self._interval_end is not None and end_ts > self._interval_end:
+                # User gave us bad endpoints; send it anyway, and let
+                # the server complain so that the error is the same
+                # as if we hadn't done this chunking.
+                end_ts = self._interval_end
+            self._block_data = [ block[spos:] ]
+            self._block_len = (epos - spos)
+            block = block[:spos]
+
+            # Next block continues where this one ended
+            self._interval_start = end_ts
+
+        # Double check endpoints
+        if start_ts is None or end_ts is None:
+            # If the block has no non-comment lines, it's OK
+            try:
+                self._get_first_noncomment(block)
+            except IndexError:
+                return
+            raise ClientError("have data to send, but no start/end times")
+
+        # Send it
+        params = { "path": self._path,
+                   "start": timestamp_to_string(start_ts),
+                   "end": timestamp_to_string(end_ts) }
+        self.last_response = self._http.put("stream/insert", block, params)
+
+        return

nilmdb/client/errors.py

@@ -0,0 +1,33 @@
+"""HTTP client errors"""
+
+from nilmdb.utils.printf import *
+
+class Error(Exception):
+    """Base exception for both ClientError and ServerError responses"""
+    def __init__(self,
+                 status = "Unspecified error",
+                 message = None,
+                 url = None,
+                 traceback = None):
+        Exception.__init__(self, status)
+        self.status = status     # e.g. "400 Bad Request"
+        self.message = message   # textual message from the server
+        self.url = url           # URL we were requesting
+        self.traceback = traceback # server traceback, if available
+    def _format_error(self, show_url):
+        s = sprintf("[%s]", self.status)
+        if self.message:
+            s += sprintf(" %s", self.message)
+        if show_url and self.url: # pragma: no cover
+            s += sprintf(" (%s)", self.url)
+        if self.traceback: # pragma: no cover
+            s += sprintf("\nServer traceback:\n%s", self.traceback)
+        return s
+    def __str__(self):
+        return self._format_error(show_url = False)
+    def __repr__(self): # pragma: no cover
+        return self._format_error(show_url = True)
+class ClientError(Error):
+    pass
+class ServerError(Error):
+    pass

nilmdb/client/httpclient.py

@@ -0,0 +1,133 @@
+"""HTTP client library"""
+
+import nilmdb.utils
+from nilmdb.client.errors import ClientError, ServerError, Error
+
+import simplejson as json
+import urlparse
+import requests
+
+class HTTPClient(object):
+    """Class to manage and perform HTTP requests from the client"""
+    def __init__(self, baseurl = "", post_json = False):
+        """If baseurl is supplied, all other functions that take
+        a URL can be given a relative URL instead."""
+        # Verify / clean up URL
+        reparsed = urlparse.urlparse(baseurl).geturl()
+        if '://' not in reparsed:
+            reparsed = urlparse.urlparse("http://" + baseurl).geturl()
+        self.baseurl = reparsed.rstrip('/') + '/'
+
+        # Build Requests session object, enable SSL verification
+        self.session = requests.Session()
+        self.session.verify = True
+
+        # Saved response, so that tests can verify a few things.
+        self._last_response = {}
+
+        # Whether to send application/json POST bodies (versus
+        # x-www-form-urlencoded)
+        self.post_json = post_json
+
+    def _handle_error(self, url, code, body):
+        # Default variables for exception.  We use the entire body as
+        # the default message, in case we can't extract it from a JSON
+        # response.
+        args = { "url" : url,
+                 "status" : str(code),
+                 "message" : body,
+                 "traceback" : None }
+        try:
+            # Fill with server-provided data if we can
+            jsonerror = json.loads(body)
+            args["status"] = jsonerror["status"]
+            args["message"] = jsonerror["message"]
+            args["traceback"] = jsonerror["traceback"]
+        except Exception: # pragma: no cover
+            pass
+        if code >= 400 and code <= 499:
+            raise ClientError(**args)
+        else: # pragma: no cover
+            if code >= 500 and code <= 599:
+                if args["message"] is None:
+                    args["message"] = ("(no message; try disabling " +
+                                       "response.stream option in " +
+                                       "nilmdb.server for better debugging)")
+                raise ServerError(**args)
+            else:
+                raise Error(**args)
+
+    def close(self):
+        self.session.close()
+
+    def _do_req(self, method, url, query_data, body_data, stream, headers):
+        url = urlparse.urljoin(self.baseurl, url)
+        try:
+            response = self.session.request(method, url,
+                                            params = query_data,
+                                            data = body_data,
+                                            stream = stream,
+                                            headers = headers)
+        except requests.RequestException as e:
+            raise ServerError(status = "502 Error", url = url,
+                              message = str(e.message))
+        if response.status_code != 200:
+            self._handle_error(url, response.status_code, response.content)
+        self._last_response = response
+        if response.headers["content-type"] in ("application/json",
+                                                "application/x-json-stream"):
+            return (response, True)
+        else:
+            return (response, False)
+
+    # Normal versions that return data directly
+    def _req(self, method, url, query = None, body = None, headers = None):
+        """
+        Make a request and return the body data as a string or parsed
+        JSON object, or raise an error if it contained an error.
+        """
+        (response, isjson) = self._do_req(method, url, query, body,
+                                          stream = False, headers = headers)
+        if isjson:
+            return json.loads(response.content)
+        return response.content
+
+    def get(self, url, params = None):
+        """Simple GET (parameters in URL)"""
+        return self._req("GET", url, params, None)
+
+    def post(self, url, params = None):
+        """Simple POST (parameters in body)"""
+        if self.post_json:
+            return self._req("POST", url, None,
+                             json.dumps(params),
+                             { 'Content-type': 'application/json' })
+        else:
+            return self._req("POST", url, None, params)
+
+    def put(self, url, data, params = None):
+        """Simple PUT (parameters in URL, data in body)"""
+        return self._req("PUT", url, params, data)
+
+    # Generator versions that return data one line at a time.
+    def _req_gen(self, method, url, query = None, body = None, headers = None):
+        """
+        Make a request and return a generator that gives back strings
+        or JSON decoded lines of the body data, or raise an error if
+        it contained an eror.
+        """
+        (response, isjson) = self._do_req(method, url, query, body,
+                                          stream = True, headers = headers)
+        if isjson:
+            for line in response.iter_lines():
+                yield json.loads(line)
+        else:
+            for line in response.iter_lines():
+                yield line
+
+    def get_gen(self, url, params = None):
+        """Simple GET (parameters in URL) returning a generator"""
+        return self._req_gen("GET", url, params)
+
+    # Not much use for a POST or PUT generator, since they don't
+    # return much data.

nilmdb/cmdline/__init__.py

@@ -1 +1,3 @@
-from .cmdline import Cmdline
+"""nilmdb.cmdline"""
+
+from nilmdb.cmdline.cmdline import Cmdline

nilmdb/cmdline/cmdline.py

@@ -1,124 +1,127 @@
 """Command line client functionality"""
 
-from __future__ import absolute_import
-from nilmdb.utils.printf import *
 import nilmdb.client
 
-import datetime_tz
-import dateutil.parser
+from nilmdb.utils.printf import *
+from nilmdb.utils import datetime_tz
+import nilmdb.utils.time
+
 import sys
-import re
+import os
 import argparse
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 
-version = "0.1"
+try: # pragma: no cover
+    import argcomplete
+except ImportError: # pragma: no cover
+    argcomplete = None
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
-                "destroy" ]
+subcommands = [ "help", "info", "create", "list", "metadata",
+                "insert", "extract", "remove", "destroy",
+                "intervals", "rename" ]
 
-# Import the subcommand modules.  Equivalent way of doing this would be
-# from . import info as cmd_info
+# Import the subcommand modules
 subcmd_mods = {}
 for cmd in subcommands:
     subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist = [ cmd ])
 
+class JimArgumentParser(argparse.ArgumentParser):
+    def error(self, message):
+        self.print_usage(sys.stderr)
+        self.exit(2, sprintf("error: %s\n", message))
+
+class Complete(object): # pragma: no cover
+    # Completion helpers, for using argcomplete (see
+    # extras/nilmtool-bash-completion.sh)
+    def escape(self, s):
+        quote_chars = [ "\\", "\"", "'", " " ]
+        for char in quote_chars:
+            s = s.replace(char, "\\" + char)
+        return s
+
+    def none(self, prefix, parsed_args, **kwargs):
+        return []
+    rate = none
+    time = none
+    url = none
+
+    def path(self, prefix, parsed_args, **kwargs):
+        client = nilmdb.client.Client(parsed_args.url)
+        return ( self.escape(s[0])
+                 for s in client.stream_list()
+                 if s[0].startswith(prefix) )
+
+    def layout(self, prefix, parsed_args, **kwargs):
+        types = [ "int8", "int16", "int32", "int64",
+                  "uint8", "uint16", "uint32", "uint64",
+                  "float32", "float64" ]
+        layouts = []
+        for i in range(1,10):
+            layouts.extend([(t + "_" + str(i)) for t in types])
+        return ( l for l in layouts if l.startswith(prefix) )
+
+    def meta_key(self, prefix, parsed_args, **kwargs):
+        return (kv.split('=')[0] for kv
+                in self.meta_keyval(prefix, parsed_args, **kwargs))
+
+    def meta_keyval(self, prefix, parsed_args, **kwargs):
+        client = nilmdb.client.Client(parsed_args.url)
+        path = parsed_args.path
+        if not path:
+            return []
+        return ( self.escape(k + '=' + v)
+                 for (k,v) in client.stream_get_metadata(path).iteritems()
+                 if k.startswith(prefix) )
+
+
 class Cmdline(object):
 
-    def __init__(self, argv):
-        self.argv = argv
+    def __init__(self, argv = None):
+        self.argv = argv or sys.argv[1:]
+        self.client = None
+        self.def_url = os.environ.get("NILMDB_URL", "http://localhost:12380")
+        self.subcmd = {}
+        self.complete = Complete()
 
     def arg_time(self, toparse):
         """Parse a time string argument"""
         try:
-            return self.parse_time(toparse).totimestamp()
+            return nilmdb.utils.time.parse_time(toparse)
         except ValueError as e:
             raise argparse.ArgumentTypeError(sprintf("%s \"%s\"",
                                                      str(e), toparse))
 
-    def parse_time(self, toparse):
-        """
-        Parse a free-form time string and return a datetime_tz object.
-        If the string doesn't contain a timestamp, the current local
-        timezone is assumed (e.g. from the TZ env var).
-        """
-        # If string doesn't contain at least 6 digits, consider it
-        # invalid.  smartparse might otherwise accept empty strings
-        # and strings with just separators.
-        if len(re.findall(r"\d", toparse)) < 6:
-            raise ValueError("not enough digits for a timestamp")
-
-        # Try to just parse the time as given
-        try:
-            return datetime_tz.datetime_tz.smartparse(toparse)
-        except ValueError:
-            pass
-
-        # Try to extract a substring in a condensed format that we expect
-        # to see in a filename or header comment
-        res = re.search(r"(^|[^\d])("            # non-numeric or SOL
-                        r"(199\d|2\d\d\d)"       # year
-                        r"[-/]?"                 # separator
-                        r"(0[1-9]|1[012])"       # month
-                        r"[-/]?"                 # separator
-                        r"([012]\d|3[01])"       # day
-                        r"[-T ]?"                # separator
-                        r"([01]\d|2[0-3])"       # hour
-                        r"[:]?"                  # separator
-                        r"([0-5]\d)"             # minute
-                        r"[:]?"                  # separator
-                        r"([0-5]\d)?"            # second
-                        r"([-+]\d\d\d\d)?"       # timezone
-                        r")", toparse)
-        if res is not None:
-            try:
-                return datetime_tz.datetime_tz.smartparse(res.group(2))
-            except ValueError:
-                pass
-
-        # Could also try to successively parse substrings, but let's
-        # just give up for now.
-        raise ValueError("unable to parse timestamp")
-
-    def time_string(self, timestamp):
-        """
-        Convert a Unix timestamp to a string for printing, using the
-        local timezone for display (e.g. from the TZ env var).
-        """
-        dt = datetime_tz.datetime_tz.fromtimestamp(timestamp)
-        return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
-
+    # Set up the parser
     def parser_setup(self):
-        version_string = sprintf("nilmtool %s, client library %s",
-                                 version, nilmdb.Client.client_version)
-
-        self.parser = argparse.ArgumentParser(add_help = False,
+        self.parser = JimArgumentParser(add_help = False,
                                         formatter_class = def_form)
 
         group = self.parser.add_argument_group("General options")
         group.add_argument("-h", "--help", action='help',
                            help='show this help message and exit')
         group.add_argument("-V", "--version", action="version",
-                           version=version_string)
+                           version = nilmdb.__version__)
 
         group = self.parser.add_argument_group("Server")
         group.add_argument("-u", "--url", action="store",
-                           default="http://localhost:12380/",
-                           help="NilmDB server URL (default: %(default)s)")
+                           default=self.def_url,
+                           help="NilmDB server URL (default: %(default)s)"
+                           ).completer = self.complete.url
 
-        sub = self.parser.add_subparsers(title="Commands",
-                                         dest="command",
-                                         description="Specify --help after "
-                                         "the command for command-specific "
-                                         "options.")
+        sub = self.parser.add_subparsers(
+            title="Commands", dest="command",
+            description="Use 'help command' or 'command --help' for more "
+            "details on a particular command.")
 
         # Set up subcommands (defined in separate files)
         for cmd in subcommands:
-            subcmd_mods[cmd].setup(self, sub)
+            self.subcmd[cmd] = subcmd_mods[cmd].setup(self, sub)
 
     def die(self, formatstr, *args):
         fprintf(sys.stderr, formatstr + "\n", *args)
+        if self.client:
             self.client.close()
         sys.exit(-1)
 
@@ -129,15 +132,23 @@ class Cmdline(object):
 
         # Run parser
         self.parser_setup()
+        if argcomplete: # pragma: no cover
+            argcomplete.autocomplete(self.parser)
         self.args = self.parser.parse_args(self.argv)
 
-        self.client = nilmdb.Client(self.args.url)
+        # Run arg verify handler if there is one
+        if "verify" in self.args:
+            self.args.verify(self)
 
-        # Make a test connection to make sure things work
+        self.client = nilmdb.client.Client(self.args.url)
+
+        # Make a test connection to make sure things work,
+        # unless the particular command requests that we don't.
+        if "no_test_connect" not in self.args:
             try:
                 server_version = self.client.version()
             except nilmdb.client.Error as e:
-            self.die("Error connecting to server: %s", str(e))
+                self.die("error connecting to server: %s", str(e))
 
         # Now dispatch client request to appropriate function.  Parser
         # should have ensured that we don't have any unknown commands

nilmdb/cmdline/create.py

@@ -1,27 +1,37 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
 import nilmdb.client
 
-from argparse import ArgumentDefaultsHelpFormatter as def_form
+from argparse import RawDescriptionHelpFormatter as raw_form
 
 def setup(self, sub):
     cmd = sub.add_parser("create", help="Create a new stream",
-                         formatter_class = def_form,
+                         formatter_class = raw_form,
                          description="""
-                         Create a new empty stream at the
-                         specified path and with the specifed
+Create a new empty stream at the specified path and with the specified
 layout type.
+
+Layout types are of the format: type_count
+
+  'type' is a data type like 'float32', 'float64', 'uint16', 'int32', etc.
+
+  'count' is the number of columns of this type.
+
+  For example, 'float32_8' means the data for this stream has 8 columns of
+  32-bit floating point values.
 """)
     cmd.set_defaults(handler = cmd_create)
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
-                       help="Path (in database) of new stream, e.g. /foo/bar")
+                       help="Path (in database) of new stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
     group.add_argument("layout",
-                       help="Layout type for new stream, e.g. float32_8")
+                       help="Layout type for new stream, e.g. float32_8",
+                       ).completer = self.complete.layout
+    return cmd
 
 def cmd_create(self):
     """Create new stream"""
     try:
         self.client.stream_create(self.args.path, self.args.layout)
     except nilmdb.client.ClientError as e:
-        self.die("Error creating stream: %s", str(e))
+        self.die("error creating stream: %s", str(e))

nilmdb/cmdline/destroy.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
 import nilmdb.client
 
@@ -8,18 +7,28 @@ def setup(self, sub):
     cmd = sub.add_parser("destroy", help="Delete a stream and all data",
                          formatter_class = def_form,
                          description="""
-                         Destroy the stream at the specified path.  All
-                         data and metadata related to the stream is
-                         permanently deleted.
+                         Destroy the stream at the specified path.
+                         The stream must be empty.  All metadata
+                         related to the stream is permanently deleted.
                          """)
     cmd.set_defaults(handler = cmd_destroy)
+    group = cmd.add_argument_group("Options")
+    group.add_argument("-R", "--remove", action="store_true",
+                       help="Remove all data before destroying stream")
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
-                       help="Path of the stream to delete, e.g. /foo/bar")
+                       help="Path of the stream to delete, e.g. /foo/bar",
+                       ).completer = self.complete.path
+    return cmd
 
 def cmd_destroy(self):
     """Destroy stream"""
+    if self.args.remove:
+        try:
+            count = self.client.stream_remove(self.args.path)
+        except nilmdb.client.ClientError as e:
+            self.die("error removing data: %s", str(e))
     try:
         self.client.stream_destroy(self.args.path)
     except nilmdb.client.ClientError as e:
-        self.die("Error deleting stream: %s", str(e))
+        self.die("error destroying stream: %s", str(e))

nilmdb/cmdline/extract.py

@@ -1,25 +1,27 @@
-from __future__ import absolute_import
 from __future__ import print_function
 from nilmdb.utils.printf import *
 import nilmdb.client
-import sys
 
 def setup(self, sub):
     cmd = sub.add_parser("extract", help="Extract data",
                          description="""
                          Extract data from a stream.
                          """)
-    cmd.set_defaults(handler = cmd_extract)
+    cmd.set_defaults(verify = cmd_extract_verify,
+                     handler = cmd_extract)
 
     group = cmd.add_argument_group("Data selection")
     group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
+                       help="Path of stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
     group.add_argument("-s", "--start", required=True,
                        metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form)")
+                       help="Starting timestamp (free-form, inclusive)",
+                       ).completer = self.complete.time
     group.add_argument("-e", "--end", required=True,
                        metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form)")
+                       help="Ending timestamp (free-form, noninclusive)",
+                       ).completer = self.complete.time
 
     group = cmd.add_argument_group("Output format")
     group.add_argument("-b", "--bare", action="store_true",
@@ -27,26 +29,42 @@ def setup(self, sub):
     group.add_argument("-a", "--annotate", action="store_true",
                        help="Include comments with some information "
                        "about the stream")
+    group.add_argument("-m", "--markup", action="store_true",
+                       help="Include comments with interval starts and ends")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps in annotated information")
     group.add_argument("-c", "--count", action="store_true",
                        help="Just output a count of matched data points")
+    return cmd
+
+def cmd_extract_verify(self):
+    if self.args.start is not None and self.args.end is not None:
+        if self.args.start > self.args.end:
+            self.parser.error("start is after end")
 
 def cmd_extract(self):
     streams = self.client.stream_list(self.args.path)
     if len(streams) != 1:
-        self.die("Error getting stream info for path %s", self.args.path)
+        self.die("error getting stream info for path %s", self.args.path)
     layout = streams[0][1]
 
+    if self.args.timestamp_raw:
+        time_string = nilmdb.utils.time.timestamp_to_string
+    else:
+        time_string = nilmdb.utils.time.timestamp_to_human
+
     if self.args.annotate:
         printf("# path: %s\n", self.args.path)
         printf("# layout: %s\n", layout)
-        printf("# start: %s\n", self.time_string(self.args.start))
-        printf("# end: %s\n", self.time_string(self.args.end))
+        printf("# start: %s\n", time_string(self.args.start))
+        printf("# end: %s\n", time_string(self.args.end))
 
     printed = False
     for dataline in self.client.stream_extract(self.args.path,
                                                self.args.start,
                                                self.args.end,
-                                               self.args.count):
+                                               self.args.count,
+                                               self.args.markup):
         if self.args.bare and not self.args.count:
             # Strip timestamp (first element).  Doesn't make sense
             # if we are only returning a count.

nilmdb/cmdline/help.py

@@ -0,0 +1,26 @@
+from nilmdb.utils.printf import *
+
+import argparse
+import sys
+
+def setup(self, sub):
+    cmd = sub.add_parser("help", help="Show detailed help for a command",
+                         description="""
+                         Show help for a command. 'help command' is
+                         the same as 'command --help'.
+                         """)
+    cmd.set_defaults(handler = cmd_help)
+    cmd.set_defaults(no_test_connect = True)
+    cmd.add_argument("command", nargs="?",
+                     help="Command to get help about")
+    cmd.add_argument("rest", nargs=argparse.REMAINDER,
+                     help=argparse.SUPPRESS)
+    return cmd
+
+def cmd_help(self):
+    if self.args.command in self.subcmd:
+        self.subcmd[self.args.command].print_help()
+    else:
+        self.parser.print_help()
+
+    return

nilmdb/cmdline/info.py

@@ -1,5 +1,6 @@
-from __future__ import absolute_import
+import nilmdb.client
 from nilmdb.utils.printf import *
+from nilmdb.utils import human_size
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 
@@ -11,11 +12,14 @@ def setup(self, sub):
                          version.
                          """)
     cmd.set_defaults(handler = cmd_info)
+    return cmd
 
 def cmd_info(self):
     """Print info about the server"""
-    printf("Client library version: %s\n", self.client.client_version)
+    printf("Client version: %s\n", nilmdb.__version__)
     printf("Server version: %s\n", self.client.version())
     printf("Server URL: %s\n", self.client.geturl())
-    printf("Server database path: %s\n", self.client.dbpath())
-    printf("Server database size: %s\n", self.client.dbsize())
+    dbinfo = self.client.dbinfo()
+    printf("Server database path: %s\n", dbinfo["path"])
+    printf("Server database size: %s\n", human_size(dbinfo["size"]))
+    printf("Server database free space: %s\n", human_size(dbinfo["free"]))

nilmdb/cmdline/insert.py

@@ -1,7 +1,7 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
 import nilmdb.client
-import nilmdb.timestamper
+import nilmdb.utils.timestamper as timestamper
+import nilmdb.utils.time
 
 import sys
 
@@ -10,96 +10,122 @@ def setup(self, sub):
                          description="""
                          Insert data into a stream.
                          """)
-    cmd.set_defaults(handler = cmd_insert)
+    cmd.set_defaults(verify = cmd_insert_verify,
+                     handler = cmd_insert)
     cmd.add_argument("-q", "--quiet", action='store_true',
                      help='suppress unnecessary messages')
 
     group = cmd.add_argument_group("Timestamping",
                                    description="""
-                                   If timestamps are already provided in the
-                                   input date, use --none.  Otherwise,
-                                   provide --start, or use --filename to
-                                   try to deduce timestamps from the file.
-
-                                   Set the TZ environment variable to change
-                                   the default timezone.
+                                   To add timestamps, specify the
+                                   arguments --timestamp and --rate,
+                                   and provide a starting time.
                                    """)
 
+    group.add_argument("-t", "--timestamp", action="store_true",
+                       help="Add timestamps to each line")
     group.add_argument("-r", "--rate", type=float,
-                       help="""
-                       If needed, rate in Hz (required when using --start)
-                       """)
+                       help="Data rate, in Hz",
+                       ).completer = self.complete.rate
+
+    group = cmd.add_argument_group("Start time",
+                                   description="""
+                                   Start time may be manually
+                                   specified with --start, or guessed
+                                   from the filenames using
+                                   --filename.  Set the TZ environment
+                                   variable to change the default
+                                   timezone.""")
+
     exc = group.add_mutually_exclusive_group()
     exc.add_argument("-s", "--start",
                      metavar="TIME", type=self.arg_time,
-                     help="Starting timestamp (free-form)")
+                     help="Starting timestamp (free-form)",
+                     ).completer = self.complete.time
     exc.add_argument("-f", "--filename", action="store_true",
-                     help="""
-                     Use filenames to determine start time
-                     (default, if filenames are provided)
-                     """)
-    exc.add_argument("-n", "--none", action="store_true",
-                     help="Timestamp is already present, don't add one")
+                     help="Use filename to determine start time")
+
+    group = cmd.add_argument_group("End time",
+                                   description="""
+                                   End time for the overall stream.
+                                   (required when not using --timestamp).
+                                   Set the TZ environment
+                                   variable to change the default
+                                   timezone.""")
+    group.add_argument("-e", "--end",
+                       metavar="TIME", type=self.arg_time,
+                       help="Ending timestamp (free-form)",
+                       ).completer = self.complete.time
 
     group = cmd.add_argument_group("Required parameters")
     group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
-    group.add_argument("file", nargs="*", default=['-'],
-                       help="File(s) to insert (default: - (stdin))")
+                       help="Path of stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
+    group.add_argument("file", nargs = '?', default='-',
+                       help="File to insert (default: - (stdin))")
+    return cmd
+
+def cmd_insert_verify(self):
+    if self.args.timestamp:
+        if not self.args.rate:
+            self.die("error: --rate is needed, but was not specified")
+        if not self.args.filename and self.args.start is None:
+            self.die("error: need --start or --filename when adding timestamps")
+    else:
+        if self.args.start is None or self.args.end is None:
+            self.die("error: when not adding timestamps, --start and "
+                     "--end are required")
 
 def cmd_insert(self):
     # Find requested stream
     streams = self.client.stream_list(self.args.path)
     if len(streams) != 1:
-        self.die("Error getting stream info for path %s", self.args.path)
+        self.die("error getting stream info for path %s", self.args.path)
 
-    layout = streams[0][1]
+    arg = self.args
 
-    if self.args.start and len(self.args.file) != 1:
-        self.die("--start can only be used with one input file, for now")
-
-    for filename in self.args.file:
+    try:
+        filename = arg.file
         if filename == '-':
             infile = sys.stdin
         else:
             try:
-                infile = open(filename, "r")
+                infile = open(filename, "rb")
             except IOError:
-                self.die("Error opening input file %s", filename)
+                self.die("error opening input file %s", filename)
 
-        # Build a timestamper for this file
-        if self.args.none:
-            ts = nilmdb.timestamper.TimestamperNull(infile)
-        else:
-            if self.args.start:
-                start = self.args.start
-            else:
+        if arg.start is None:
             try:
-                    start = self.parse_time(filename)
+                arg.start = nilmdb.utils.time.parse_time(filename)
             except ValueError:
-                    self.die("Error extracting time from filename '%s'",
+                self.die("error extracting start time from filename '%s'",
                          filename)
 
-            if not self.args.rate:
-                self.die("Need to specify --rate")
-            rate = self.args.rate
-
-            ts = nilmdb.timestamper.TimestamperRate(infile, start, rate)
+        if arg.timestamp:
+            data = timestamper.TimestamperRate(infile, arg.start, arg.rate)
+        else:
+            data = iter(lambda: infile.read(1048576), '')
 
         # Print info
-        if not self.args.quiet:
+        if not arg.quiet:
             printf(" Input file: %s\n", filename)
-            printf("Timestamper: %s\n", str(ts))
+            printf(" Start time: %s\n",
+                   nilmdb.utils.time.timestamp_to_human(arg.start))
+            if arg.end:
+                printf("   End time: %s\n",
+                       nilmdb.utils.time.timestamp_to_human(arg.end))
+            if arg.timestamp:
+                printf("Timestamper: %s\n", str(data))
 
         # Insert the data
-        try:
-            result = self.client.stream_insert(self.args.path, ts)
+        self.client.stream_insert(arg.path, data, arg.start, arg.end)
+
     except nilmdb.client.Error as e:
         # TODO: It would be nice to be able to offer better errors
         # here, particularly in the case of overlap, which just shows
         # ugly bracketed ranges of 16-digit numbers and a mangled URL.
         # Need to consider adding something like e.prettyprint()
         # that is smarter about the contents of the error.
-            self.die("Error inserting data: %s", str(e))
+        self.die("error inserting data: %s", str(e))
 
     return

nilmdb/cmdline/intervals.py

@@ -0,0 +1,66 @@
+from nilmdb.utils.printf import *
+import nilmdb.utils.time
+
+import fnmatch
+import argparse
+from argparse import ArgumentDefaultsHelpFormatter as def_form
+
+def setup(self, sub):
+    cmd = sub.add_parser("intervals", help="List intervals",
+                         formatter_class = def_form,
+                         description="""
+                         List intervals in a stream, similar to
+                         'list --detail path'.
+
+                         If '--diff diffpath' is provided, only
+                         interval ranges that are present in 'path'
+                         and not present in 'diffpath' are printed.
+                         """)
+    cmd.set_defaults(verify = cmd_intervals_verify,
+                     handler = cmd_intervals)
+
+    group = cmd.add_argument_group("Stream selection")
+    group.add_argument("path", metavar="PATH",
+                       help="List intervals for this path",
+                       ).completer = self.complete.path
+    group.add_argument("-d", "--diff", metavar="PATH",
+                       help="Subtract intervals from this path",
+                       ).completer = self.complete.path
+
+    group = cmd.add_argument_group("Interval details")
+    group.add_argument("-s", "--start",
+                       metavar="TIME", type=self.arg_time,
+                       help="Starting timestamp for intervals "
+                       "(free-form, inclusive)",
+                       ).completer = self.complete.time
+    group.add_argument("-e", "--end",
+                       metavar="TIME", type=self.arg_time,
+                       help="Ending timestamp for intervals "
+                       "(free-form, noninclusive)",
+                       ).completer = self.complete.time
+
+    group = cmd.add_argument_group("Misc options")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps when printing times")
+
+    return cmd
+
+def cmd_intervals_verify(self):
+    if self.args.start is not None and self.args.end is not None:
+        if self.args.start >= self.args.end:
+            self.parser.error("start must precede end")
+
+def cmd_intervals(self):
+    """List intervals in a stream"""
+    if self.args.timestamp_raw:
+        time_string = nilmdb.utils.time.timestamp_to_string
+    else:
+        time_string = nilmdb.utils.time.timestamp_to_human
+
+    try:
+           for (start, end) in self.client.stream_intervals(
+               self.args.path, self.args.start, self.args.end, self.args.diff):
+               printf("[ %s -> %s ]\n", time_string(start), time_string(end))
+    except nilmdb.client.ClientError as e:
+        self.die("error listing intervals: %s", str(e))
+

nilmdb/cmdline/list.py

@@ -1,8 +1,8 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
-import nilmdb.client
+import nilmdb.utils.time
 
 import fnmatch
+import argparse
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 
 def setup(self, sub):
@@ -13,42 +13,96 @@ def setup(self, sub):
                          optionally filtering by layout or path.  Wildcards
                          are accepted.
                          """)
-    cmd.set_defaults(handler = cmd_list)
+    cmd.set_defaults(verify = cmd_list_verify,
+                     handler = cmd_list)
 
     group = cmd.add_argument_group("Stream filtering")
+    group.add_argument("-p", "--path", metavar="PATH", default="*",
+                       help="Match only this path (-p can be omitted)",
+                       ).completer = self.complete.path
+    group.add_argument("path_positional", default="*",
+                       nargs="?", help=argparse.SUPPRESS,
+                       ).completer = self.complete.path
     group.add_argument("-l", "--layout", default="*",
-                       help="Match only this stream layout")
-    group.add_argument("-p", "--path", default="*",
-                       help="Match only this path")
+                       help="Match only this stream layout",
+                       ).completer = self.complete.layout
+
+    group = cmd.add_argument_group("Interval info")
+    group.add_argument("-E", "--ext", action="store_true",
+                       help="Show extended stream info, like interval "
+                       "extents and row count")
 
     group = cmd.add_argument_group("Interval details")
     group.add_argument("-d", "--detail", action="store_true",
                        help="Show available data time intervals")
     group.add_argument("-s", "--start",
                        metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form)")
+                       help="Starting timestamp for intervals "
+                       "(free-form, inclusive)",
+                       ).completer = self.complete.time
     group.add_argument("-e", "--end",
                        metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form)")
+                       help="Ending timestamp for intervals "
+                       "(free-form, noninclusive)",
+                       ).completer = self.complete.time
+
+    group = cmd.add_argument_group("Misc options")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps when printing times")
+
+    return cmd
+
+def cmd_list_verify(self):
+    # A hidden "path_positional" argument lets the user leave off the
+    # "-p" when specifying the path.  Handle it here.
+    got_opt = self.args.path != "*"
+    got_pos = self.args.path_positional != "*"
+    if got_pos:
+        if got_opt:
+            self.parser.error("too many paths specified")
+        else:
+            self.args.path = self.args.path_positional
+
+    if self.args.start is not None and self.args.end is not None:
+        if self.args.start >= self.args.end:
+            self.parser.error("start must precede end")
+
+    if self.args.start is not None or self.args.end is not None:
+        if not self.args.detail:
+            self.parser.error("--start and --end only make sense with --detail")
 
 def cmd_list(self):
     """List available streams"""
-    streams = self.client.stream_list()
-    for (path, layout) in streams:
+    streams = self.client.stream_list(extended = True)
+
+    if self.args.timestamp_raw:
+        time_string = nilmdb.utils.time.timestamp_to_string
+    else:
+        time_string = nilmdb.utils.time.timestamp_to_human
+
+    for stream in streams:
+        (path, layout, int_min, int_max, rows, time) = stream[:6]
         if not (fnmatch.fnmatch(path, self.args.path) and
                 fnmatch.fnmatch(layout, self.args.layout)):
             continue
 
         printf("%s %s\n", path, layout)
-        if not self.args.detail:
-            continue
 
+        if self.args.ext:
+            if int_min is None or int_max is None:
+                printf("  interval extents: (no data)\n")
+            else:
+                printf("  interval extents: %s -> %s\n",
+                       time_string(int_min), time_string(int_max))
+            printf("        total data: %d rows, %.6f seconds\n",
+                   rows or 0,
+                   nilmdb.utils.time.timestamp_to_seconds(time or 0))
+
+        if self.args.detail:
             printed = False
-        for (start, end) in self.client.stream_intervals(path, self.args.start,
-                                                         self.args.end):
-            printf("  [ %s -> %s ]\n",
-                   self.time_string(start),
-                   self.time_string(end))
+            for (start, end) in self.client.stream_intervals(
+                path, self.args.start, self.args.end):
+                printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
                 printed = True
             if not printed:
                 printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 
 def setup(self, sub):
@@ -14,18 +14,23 @@ def setup(self, sub):
 
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
+                       help="Path of stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
 
     group = cmd.add_argument_group("Actions")
     exc = group.add_mutually_exclusive_group()
     exc.add_argument("-g", "--get", nargs="*", metavar="key",
-                     help="Get metadata for specified keys (default all)")
+                     help="Get metadata for specified keys (default all)",
+                     ).completer = self.complete.meta_key
     exc.add_argument("-s", "--set", nargs="+", metavar="key=value",
                      help="Replace all metadata with provided "
-                     "key=value pairs")
+                     "key=value pairs",
+                     ).completer = self.complete.meta_keyval
     exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
                      help="Update metadata using provided "
-                     "key=value pairs")
+                     "key=value pairs",
+                     ).completer = self.complete.meta_keyval
+    return cmd
 
 def cmd_metadata(self):
     """Manipulate metadata"""
@@ -43,21 +48,21 @@ def cmd_metadata(self):
         for keyval in keyvals:
             kv = keyval.split('=')
             if len(kv) != 2 or kv[0] == "":
-                self.die("Error parsing key=value argument '%s'", keyval)
+                self.die("error parsing key=value argument '%s'", keyval)
             data[kv[0]] = kv[1]
 
         # Make the call
         try:
             handler(self.args.path, data)
         except nilmdb.client.ClientError as e:
-            self.die("Error setting/updating metadata: %s", str(e))
+            self.die("error setting/updating metadata: %s", str(e))
     else:
         # Get (or unspecified)
         keys = self.args.get or None
         try:
             data = self.client.stream_get_metadata(self.args.path, keys)
         except nilmdb.client.ClientError as e:
-            self.die("Error getting metadata: %s", str(e))
+            self.die("error getting metadata: %s", str(e))
         for key, value in sorted(data.items()):
             # Omit nonexistant keys
             if value is None:

nilmdb/cmdline/remove.py

@@ -0,0 +1,40 @@
+from nilmdb.utils.printf import *
+import nilmdb.client
+
+def setup(self, sub):
+    cmd = sub.add_parser("remove", help="Remove data",
+                         description="""
+                         Remove all data from a specified time range within a
+                         stream.
+                         """)
+    cmd.set_defaults(handler = cmd_remove)
+
+    group = cmd.add_argument_group("Data selection")
+    group.add_argument("path",
+                       help="Path of stream, e.g. /foo/bar",
+                       ).completer = self.complete.path
+    group.add_argument("-s", "--start", required=True,
+                       metavar="TIME", type=self.arg_time,
+                       help="Starting timestamp (free-form, inclusive)",
+                       ).completer = self.complete.time
+    group.add_argument("-e", "--end", required=True,
+                       metavar="TIME", type=self.arg_time,
+                       help="Ending timestamp (free-form, noninclusive)",
+                       ).completer = self.complete.time
+
+    group = cmd.add_argument_group("Output format")
+    group.add_argument("-c", "--count", action="store_true",
+                       help="Output number of data points removed")
+    return cmd
+
+def cmd_remove(self):
+    try:
+        count = self.client.stream_remove(self.args.path,
+                                          self.args.start, self.args.end)
+    except nilmdb.client.ClientError as e:
+        self.die("error removing data: %s", str(e))
+
+    if self.args.count:
+        printf("%d\n", count)
+
+    return 0

nilmdb/cmdline/rename.py

@@ -0,0 +1,31 @@
+from nilmdb.utils.printf import *
+import nilmdb.client
+
+from argparse import ArgumentDefaultsHelpFormatter as def_form
+
+def setup(self, sub):
+    cmd = sub.add_parser("rename", help="Rename a stream",
+                         formatter_class = def_form,
+                         description="""
+                         Rename a stream.
+
+                         Only the stream's path is renamed; no
+                         metadata is changed.
+                         """)
+    cmd.set_defaults(handler = cmd_rename)
+    group = cmd.add_argument_group("Required arguments")
+    group.add_argument("oldpath",
+                       help="Old path, e.g. /foo/old",
+                       ).completer = self.complete.path
+    group.add_argument("newpath",
+                       help="New path, e.g. /foo/bar/new",
+                       ).completer = self.complete.path
+
+    return cmd
+
+def cmd_rename(self):
+    """Rename a stream"""
+    try:
+        self.client.stream_rename(self.args.oldpath, self.args.newpath)
+    except nilmdb.client.ClientError as e:
+        self.die("error renaming stream: %s", str(e))

nilmdb/httpclient.py

@@ -1,223 +0,0 @@
-"""HTTP client library"""
-
-from __future__ import absolute_import
-from nilmdb.utils.printf import *
-import nilmdb.utils
-
-import time
-import sys
-import re
-import os
-import simplejson as json
-import urlparse
-import pycurl
-import cStringIO
-
-class Error(Exception):
-    """Base exception for both ClientError and ServerError responses"""
-    def __init__(self,
-                 status = "Unspecified error",
-                 message = None,
-                 url = None,
-                 traceback = None):
-        Exception.__init__(self, status)
-        self.status = status     # e.g. "400 Bad Request"
-        self.message = message   # textual message from the server
-        self.url = url           # URL we were requesting
-        self.traceback = traceback # server traceback, if available
-    def __str__(self):
-        s = sprintf("[%s]", self.status)
-        if self.message:
-            s += sprintf(" %s", self.message)
-        if self.url:
-            s += sprintf(" (%s)", self.url)
-        if self.traceback: # pragma: no cover
-            s += sprintf("\nServer traceback:\n%s", self.traceback)
-        return s
-class ClientError(Error):
-    pass
-class ServerError(Error):
-    pass
-
-class HTTPClient(object):
-    """Class to manage and perform HTTP requests from the client"""
-    def __init__(self, baseurl = ""):
-        """If baseurl is supplied, all other functions that take
-        a URL can be given a relative URL instead."""
-        # Verify / clean up URL
-        reparsed = urlparse.urlparse(baseurl).geturl()
-        if '://' not in reparsed:
-            reparsed = urlparse.urlparse("http://" + baseurl).geturl()
-        self.baseurl = reparsed
-        self.curl = pycurl.Curl()
-        self.curl.setopt(pycurl.SSL_VERIFYHOST, 2)
-        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
-        self.curl.setopt(pycurl.MAXREDIRS, 5)
-        self._setup_url()
-
-    def _setup_url(self, url = "", params = ""):
-        url = urlparse.urljoin(self.baseurl, url)
-        if params:
-            url = urlparse.urljoin(
-                url, "?" + nilmdb.utils.urllib.urlencode(params, True))
-        self.curl.setopt(pycurl.URL, url)
-        self.url = url
-
-    def _check_error(self, body = None):
-        code = self.curl.getinfo(pycurl.RESPONSE_CODE)
-        if code == 200:
-            return
-        # Default variables for exception
-        args = { "url" : self.url,
-                 "status" : str(code),
-                 "message" : None,
-                 "traceback" : None }
-        try:
-            # Fill with server-provided data if we can
-            jsonerror = json.loads(body)
-            args["status"] = jsonerror["status"]
-            args["message"] = jsonerror["message"]
-            args["traceback"] = jsonerror["traceback"]
-        except Exception: # pragma: no cover
-            pass
-        if code >= 400 and code <= 499:
-            raise ClientError(**args)
-        else: # pragma: no cover
-            if code >= 500 and code <= 599:
-                if args["message"] is None:
-                    args["message"] = ("(no message; try disabling " +
-                                       "response.stream option in " +
-                                       "nilmdb.server for better debugging)")
-                raise ServerError(**args)
-            else:
-                raise Error(**args)
-
-    def _req_generator(self, url, params):
-        """
-        Like self._req(), but runs the perform in a separate thread.
-        It returns a generator that spits out arbitrary-sized chunks
-        of the resulting data, instead of using the WRITEFUNCTION
-        callback.
-        """
-        self._setup_url(url, params)
-        self._status = None
-        error_body = ""
-        self._headers = ""
-        def header_callback(data):
-            if self._status is None:
-                self._status = int(data.split(" ")[1])
-            self._headers += data
-        self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
-        def func(callback):
-            self.curl.setopt(pycurl.WRITEFUNCTION, callback)
-            self.curl.perform()
-        try:
-            for i in nilmdb.utils.Iteratorizer(func):
-                if self._status == 200:
-                    # If we had a 200 response, yield the data to the caller.
-                    yield i
-                else:
-                    # Otherwise, collect it into an error string.
-                    error_body += i
-        except pycurl.error as e:
-            raise ServerError(status = "502 Error",
-                              url = self.url,
-                              message = e[1])
-        # Raise an exception if there was an error
-        self._check_error(error_body)
-
-    def _req(self, url, params):
-        """
-        GET or POST that returns raw data.  Returns the body
-        data as a string, or raises an error if it contained an error.
-        """
-        self._setup_url(url, params)
-        body = cStringIO.StringIO()
-        self.curl.setopt(pycurl.WRITEFUNCTION, body.write)
-        self._headers = ""
-        def header_callback(data):
-            self._headers += data
-        self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
-        try:
-            self.curl.perform()
-        except pycurl.error as e:
-            raise ServerError(status = "502 Error",
-                              url = self.url,
-                              message = e[1])
-        body_str = body.getvalue()
-        # Raise an exception if there was an error
-        self._check_error(body_str)
-        return body_str
-
-    def close(self):
-        self.curl.close()
-
-    def _iterate_lines(self, it):
-        """
-        Given an iterator that returns arbitrarily-sized chunks
-        of data, return '\n'-delimited lines of text
-        """
-        partial = ""
-        for chunk in it:
-            partial += chunk
-            lines = partial.split("\n")
-            for line in lines[0:-1]:
-                yield line
-            partial = lines[-1]
-        if partial != "":
-            yield partial
-
-    # Non-generator versions
-    def _doreq(self, url, params, retjson):
-        """
-        Perform a request, and return the body.
-
-        url: URL to request (relative to baseurl)
-        params: dictionary of query parameters
-        retjson: expect JSON and return python objects instead of string
-        """
-        out = self._req(url, params)
-        if retjson:
-            return json.loads(out)
-        return out
-
-    def get(self, url, params = None, retjson = True):
-        """Simple GET"""
-        self.curl.setopt(pycurl.UPLOAD, 0)
-        return self._doreq(url, params, retjson)
-
-    def put(self, url, postdata, params = None, retjson = True):
-        """Simple PUT"""
-        self._setup_url(url, params)
-        data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.UPLOAD, 1)
-        self.curl.setopt(pycurl.READFUNCTION, data.read)
-        return self._doreq(url, params, retjson)
-
-    # Generator versions
-    def _doreq_gen(self, url, params, retjson):
-        """
-        Perform a request, and return lines of the body in a generator.
-
-        url: URL to request (relative to baseurl)
-        params: dictionary of query parameters
-        retjson: expect JSON and yield python objects instead of strings
-        """
-        for line in self._iterate_lines(self._req_generator(url, params)):
-            if retjson:
-                yield json.loads(line)
-            else:
-                yield line
-
-    def get_gen(self, url, params = None, retjson = True):
-        """Simple GET, returning a generator"""
-        self.curl.setopt(pycurl.UPLOAD, 0)
-        return self._doreq_gen(url, params, retjson)
-
-    def put_gen(self, url, postdata, params = None, retjson = True):
-        """Simple PUT, returning a generator"""
-        self._setup_url(url, params)
-        data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.UPLOAD, 1)
-        self.curl.setopt(pycurl.READFUNCTION, data.read)
-        return self._doreq_gen(url, params, retjson)

nilmdb/layout.pyx

@@ -1,209 +0,0 @@
-# cython: profile=False
-
-import time
-import sys
-import inspect
-import cStringIO
-import numpy as np
-
-cdef enum:
-    max_value_count = 64
-
-cimport cython
-cimport libc.stdlib
-cimport libc.stdio
-cimport libc.string
-
-class ParserError(Exception):
-    def __init__(self, line, message):
-        self.message = "line " + str(line) + ": " + message
-        Exception.__init__(self, self.message)
-
-class FormatterError(Exception):
-    pass
-
-class Layout:
-    """Represents a NILM database layout"""
-
-    def __init__(self, typestring):
-        """Initialize this Layout object to handle the specified
-        type string"""
-        try:
-            [ datatype, count ] = typestring.split("_")
-        except:
-            raise KeyError("invalid layout string")
-
-        try:
-            self.count = int(count)
-        except ValueError:
-            raise KeyError("invalid count")
-        if self.count < 1 or self.count > max_value_count:
-            raise KeyError("invalid count")
-
-        if datatype == 'uint16':
-            self.parse = self.parse_uint16
-            self.format = self.format_uint16
-        elif datatype == 'float32' or datatype == 'float64':
-            self.parse = self.parse_float64
-            self.format = self.format_float64
-        else:
-            raise KeyError("invalid type")
-
-        self.datatype = datatype
-
-    # Parsers
-    def parse_float64(self, char *text):
-        cdef int n
-        cdef double ts
-        # Return doubles even in float32 case, since they're going into
-        # a Python array which would upconvert to double anyway.
-        result = []
-        cdef char *end
-        ts = libc.stdlib.strtod(text, &end)
-        if end == text:
-            raise ValueError("bad timestamp")
-        result.append(ts)
-        for n in range(self.count):
-            text = end
-            result.append(libc.stdlib.strtod(text, &end))
-            if end == text:
-                raise ValueError("wrong number of values")
-        n = 0
-        while end[n] == ' ':
-            n += 1
-        if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
-            raise ValueError("extra data on line")
-        return (ts, result)
-
-    def parse_uint16(self, char *text):
-        cdef int n
-        cdef double ts
-        cdef int v
-        result = []
-        cdef char *end
-        ts = libc.stdlib.strtod(text, &end)
-        if end == text:
-            raise ValueError("bad timestamp")
-        result.append(ts)
-        for n in range(self.count):
-            text = end
-            v = libc.stdlib.strtol(text, &end, 10)
-            if v < 0 or v > 65535:
-                raise ValueError("value out of range")
-            result.append(v)
-            if end == text:
-                raise ValueError("wrong number of values")
-        n = 0
-        while end[n] == ' ':
-            n += 1
-        if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
-            raise ValueError("extra data on line")
-        return (ts, result)
-
-    # Formatters
-    def format_float64(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %f" % d[i+1]
-        return s + "\n"
-
-    def format_uint16(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %d" % d[i+1]
-        return s + "\n"
-
-# Get a layout by name
-def get_named(typestring):
-    try:
-        return Layout(typestring)
-    except KeyError:
-        compat = { "PrepData": "float32_8",
-                   "RawData": "uint16_6",
-                   "RawNotchedData": "uint16_9" }
-        return Layout(compat[typestring])
-
-class Parser(object):
-    """Object that parses and stores ASCII data for inclusion into the
-    database"""
-
-    def __init__(self, layout):
-        if issubclass(layout.__class__, Layout):
-            self.layout = layout
-        else:
-            try:
-                self.layout = get_named(layout)
-            except KeyError:
-                raise TypeError("unknown layout")
-
-        self.data = []
-        self.min_timestamp = None
-        self.max_timestamp = None
-
-    def parse(self, textdata):
-        """
-        Parse the data, provided as lines of text, using the current
-        layout, into an internal data structure suitable for a
-        pytables 'table.append(parser.data)'.
-        """
-        cdef double last_ts = 0, ts
-        cdef int n = 0, i
-        cdef char *line
-
-        indata = cStringIO.StringIO(textdata)
-        # Assume any parsing error is a real error.
-        # In the future we might want to skip completely empty lines,
-        # or partial lines right before EOF?
-        try:
-            self.data = []
-            for pyline in indata:
-                line = pyline
-                n += 1
-                if line[0] == '\#':
-                    continue
-                (ts, row) = self.layout.parse(line)
-                if ts < last_ts:
-                    raise ValueError("timestamp is not "
-                                     "monotonically increasing")
-                last_ts = ts
-                self.data.append(row)
-        except (ValueError, IndexError, TypeError) as e:
-            raise ParserError(n, "error: " + e.message)
-
-        # Mark timestamp ranges
-        if len(self.data):
-            self.min_timestamp = self.data[0][0]
-            self.max_timestamp = self.data[-1][0]
-
-class Formatter(object):
-    """Object that formats database data into ASCII"""
-
-    def __init__(self, layout):
-        if issubclass(layout.__class__, Layout):
-            self.layout = layout
-        else:
-            try:
-                self.layout = get_named(layout)
-            except KeyError:
-                raise TypeError("unknown layout")
-
-    def format(self, data):
-        """
-        Format raw data from the database, using the current layout,
-        as lines of ACSII text.
-        """
-        text = cStringIO.StringIO()
-        try:
-            for row in data:
-                text.write(self.layout.format(row))
-        except (ValueError, IndexError, TypeError) as e:
-            raise FormatterError("formatting error: " + e.message)
-        return text.getvalue()

nilmdb/nilmdb.py

@@ -1,474 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""NilmDB
-
-Object that represents a NILM database file.
-
-Manages both the SQL database and the table storage backend.
-"""
-
-# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
-# but will pull the nilmdb module instead.
-from __future__ import absolute_import
-import nilmdb
-from nilmdb.utils.printf import *
-
-import sqlite3
-import time
-import sys
-import os
-import errno
-import bisect
-
-import pyximport
-pyximport.install()
-from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
-
-from . import bulkdata
-
-# Note about performance and transactions:
-#
-# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
-# takes about 125msec.  sqlite3 will commit transactions at 3 times:
-# 1: explicit con.commit()
-# 2: between a series of DML commands and non-DML commands, e.g.
-#    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
-# 3: at the end of an explicit transaction, e.g. "with self.con as con:"
-#
-# To speed up testing, or if this transaction speed becomes an issue,
-# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF.
-
-
-# Don't touch old entries -- just add new ones.
-_sql_schema_updates = {
-    0: """
-    -- All streams
-    CREATE TABLE streams(
-    	id INTEGER PRIMARY KEY,		-- stream ID
-        path TEXT UNIQUE NOT NULL,	-- path, e.g. '/newton/prep'
-        layout TEXT NOT NULL		-- layout name, e.g. float32_8
-    );
-
-    -- Individual timestamped ranges in those streams.
-    -- For a given start_time and end_time, this tells us that the
-    -- data is stored between start_pos and end_pos.
-    -- Times are stored as μs since Unix epoch
-    -- Positions are opaque: PyTables rows, file offsets, etc.
-    --
-    -- Note: end_pos points to the row _after_ end_time, so end_pos-1
-    -- is the last valid row.
-    CREATE TABLE ranges(
-        stream_id INTEGER NOT NULL,
-        start_time INTEGER NOT NULL,
-        end_time INTEGER NOT NULL,
-        start_pos INTEGER NOT NULL,
-        end_pos INTEGER NOT NULL
-    );
-    CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time);
-    """,
-
-    1: """
-    -- Generic dictionary-type metadata that can be associated with a stream
-    CREATE TABLE metadata(
-    	stream_id INTEGER NOT NULL,
-        key TEXT NOT NULL,
-        value TEXT
-    );
-    """,
-}
-
-class NilmDBError(Exception):
-    """Base exception for NilmDB errors"""
-    def __init__(self, message = "Unspecified error"):
-        Exception.__init__(self, self.__class__.__name__ + ": " + message)
-
-class StreamError(NilmDBError):
-    pass
-
-class OverlapError(NilmDBError):
-    pass
-
-@nilmdb.utils.must_close()
-class NilmDB(object):
-    verbose = 0
-
-    def __init__(self, basepath, sync=True, max_results=None):
-        # set up path
-        self.basepath = os.path.abspath(basepath)
-
-        # Create the database path if it doesn't exist
-        try:
-            os.makedirs(self.basepath)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise IOError("can't create tree " + self.basepath)
-
-        # Our data goes inside it
-        self.data = bulkdata.BulkData(self.basepath)
-
-        # SQLite database too
-        sqlfilename = os.path.join(self.basepath, "data.sql")
-        # We use check_same_thread = False, assuming that the rest
-        # of the code (e.g. Server) will be smart and not access this
-        # database from multiple threads simultaneously.  Otherwise
-        # false positives will occur when the database is only opened
-        # in one thread, and only accessed in another.
-        self.con = sqlite3.connect(sqlfilename, check_same_thread = False)
-        self._sql_schema_update()
-
-        # See big comment at top about the performance implications of this
-        if sync:
-            self.con.execute("PRAGMA synchronous=FULL")
-        else:
-            self.con.execute("PRAGMA synchronous=OFF")
-
-        # Approximate largest number of elements that we want to send
-        # in a single reply (for stream_intervals, stream_extract)
-        if max_results:
-            self.max_results = max_results
-        else:
-            self.max_results = 16384
-
-    def get_basepath(self):
-        return self.basepath
-
-    def close(self):
-        if self.con:
-            self.con.commit()
-            self.con.close()
-        self.data.close()
-
-    def _sql_schema_update(self):
-        cur = self.con.cursor()
-        version = cur.execute("PRAGMA user_version").fetchone()[0]
-        oldversion = version
-
-        while version in _sql_schema_updates:
-            cur.executescript(_sql_schema_updates[version])
-            version = version + 1
-            if self.verbose: # pragma: no cover
-                printf("Schema updated to %d\n", version)
-
-        if version != oldversion:
-            with self.con:
-                cur.execute("PRAGMA user_version = {v:d}".format(v=version))
-
-    @nilmdb.utils.lru_cache(size = 16)
-    def _get_intervals(self, stream_id):
-        """
-        Return a mutable IntervalSet corresponding to the given stream ID.
-        """
-        iset = IntervalSet()
-        result = self.con.execute("SELECT start_time, end_time, "
-                                  "start_pos, end_pos "
-                                  "FROM ranges "
-                                  "WHERE stream_id=?", (stream_id,))
-        try:
-            for (start_time, end_time, start_pos, end_pos) in result:
-                iset += DBInterval(start_time, end_time,
-                                   start_time, end_time,
-                                   start_pos, end_pos)
-        except IntervalError as e: # pragma: no cover
-            raise NilmDBError("unexpected overlap in ranges table!")
-
-        return iset
-
-    def _add_interval(self, stream_id, interval, start_pos, end_pos):
-        """
-        Add interval to the internal interval cache, and to the database.
-        Note: arguments must be ints (not numpy.int64, etc)
-        """
-        # Load this stream's intervals
-        iset = self._get_intervals(stream_id)
-
-        # Check for overlap
-        if iset.intersects(interval): # pragma: no cover (gets caught earlier)
-            raise NilmDBError("new interval overlaps existing data")
-
-        # Check for adjacency.  If there's a stream in the database
-        # that ends exactly when this one starts, and the database
-        # rows match up, we can make one interval that covers the
-        # time range [adjacent.start -> interval.end)
-        # and database rows [ adjacent.start_pos -> end_pos ].
-        # Only do this if the resulting interval isn't too large.
-        max_merged_rows = 30000000 # a bit more than 1 hour at 8 KHz
-        adjacent = iset.find_end(interval.start)
-        if (adjacent is not None and
-            start_pos == adjacent.db_endpos and
-            (end_pos - adjacent.db_startpos) < max_merged_rows):
-            # First delete the old one, both from our iset and the
-            # database
-            iset -= adjacent
-            self.con.execute("DELETE FROM ranges WHERE "
-                             "stream_id=? AND start_time=? AND "
-                             "end_time=? AND start_pos=? AND "
-                             "end_pos=?", (stream_id,
-                                           adjacent.db_start,
-                                           adjacent.db_end,
-                                           adjacent.db_startpos,
-                                           adjacent.db_endpos))
-
-            # Now update our interval so the fallthrough add is
-            # correct.
-            interval.start = adjacent.start
-            start_pos = adjacent.db_startpos
-
-        # Add the new interval to the iset
-        iset.iadd_nocheck(DBInterval(interval.start, interval.end,
-                                     interval.start, interval.end,
-                                     start_pos, end_pos))
-
-        # Insert into the database
-        self.con.execute("INSERT INTO ranges "
-                         "(stream_id,start_time,end_time,start_pos,end_pos) "
-                         "VALUES (?,?,?,?,?)",
-                         (stream_id, interval.start, interval.end,
-                          int(start_pos), int(end_pos)))
-
-        self.con.commit()
-
-    def stream_list(self, path = None, layout = None):
-        """Return list of [path, layout] lists of all streams
-        in the database.
-
-        If path is specified, include only streams with a path that
-        matches the given string.
-
-        If layout is specified, include only streams with a layout
-        that matches the given string.
-        """
-        where = "WHERE 1=1"
-        params = ()
-        if layout:
-            where += " AND layout=?"
-            params += (layout,)
-        if path:
-            where += " AND path=?"
-            params += (path,)
-        result = self.con.execute("SELECT path, layout "
-                                  "FROM streams " + where, params).fetchall()
-
-        return sorted(list(x) for x in result)
-
-    def stream_intervals(self, path, start = None, end = None):
-        """
-        Returns (intervals, restart) tuple.
-
-        intervals is a list of [start,end] timestamps of all intervals
-        that exist for path, between start and end.
-
-        restart, if nonzero, means that there were too many results to
-        return in a single request.  The data is complete from the
-        starting timestamp to the point at which it was truncated,
-        and a new request with a start time of 'restart' will fetch
-        the next block of data.
-        """
-        stream_id = self._stream_id(path)
-        intervals = self._get_intervals(stream_id)
-        requested = Interval(start or 0, end or 1e12)
-        result = []
-        for n, i in enumerate(intervals.intersection(requested)):
-            if n >= self.max_results:
-                restart = i.start
-                break
-            result.append([i.start, i.end])
-        else:
-            restart = 0
-        return (result, restart)
-
-    def stream_create(self, path, layout_name):
-        """Create a new table in the database.
-
-        path: path to the data (e.g. '/newton/prep').
-        Paths must contain at least two elements, e.g.:
-           /newton/prep
-           /newton/raw
-           /newton/upstairs/prep
-           /newton/upstairs/raw
-
-        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
-        """
-        # Create the bulk storage.  Raises ValueError on error, which we
-        # pass along.
-        self.data.create(path, layout_name)
-
-        # Insert into SQL database once the bulk storage is happy
-        with self.con as con:
-            con.execute("INSERT INTO streams (path, layout) VALUES (?,?)",
-                        (path, layout_name))
-
-    def _stream_id(self, path):
-        """Return unique stream ID"""
-        result = self.con.execute("SELECT id FROM streams WHERE path=?",
-                                  (path,)).fetchone()
-        if result is None:
-            raise StreamError("No stream at path " + path)
-        return result[0]
-
-    def stream_set_metadata(self, path, data):
-        """Set stream metadata from a dictionary, e.g.
-           { description = 'Downstairs lighting',
-             v_scaling = 123.45 }
-           This replaces all existing metadata.
-           """
-        stream_id = self._stream_id(path)
-        with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
-            for key in data:
-                if data[key] != '':
-                    con.execute("INSERT INTO metadata VALUES (?, ?, ?)",
-                                (stream_id, key, data[key]))
-
-    def stream_get_metadata(self, path):
-        """Return stream metadata as a dictionary."""
-        stream_id = self._stream_id(path)
-        result = self.con.execute("SELECT metadata.key, metadata.value "
-                                  "FROM metadata "
-                                  "WHERE metadata.stream_id=?", (stream_id,))
-        data = {}
-        for (key, value) in result:
-            data[key] = value
-        return data
-
-    def stream_update_metadata(self, path, newdata):
-        """Update stream metadata from a dictionary"""
-        data = self.stream_get_metadata(path)
-        data.update(newdata)
-        self.stream_set_metadata(path, data)
-
-    def stream_destroy(self, path):
-        """Fully remove a table and all of its data from the database.
-        No way to undo it!  Metadata is removed."""
-        stream_id = self._stream_id(path)
-
-        # Delete the cached interval data
-        self._get_intervals.cache_remove(self, stream_id)
-
-        # Delete the data
-        self.data.destroy(path)
-
-        # Delete metadata, stream, intervals
-        with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
-
-    def stream_insert(self, path, start, end, data):
-        """Insert new data into the database.
-           path: Path at which to add the data
-           start: Starting timestamp
-           end: Ending timestamp
-           data: Rows of data, to be passed to PyTable's table.append
-                 method.  E.g. nilmdb.layout.Parser.data
-           """
-        # First check for basic overlap using timestamp info given.
-        stream_id = self._stream_id(path)
-        iset = self._get_intervals(stream_id)
-        interval = Interval(start, end)
-        if iset.intersects(interval):
-            raise OverlapError("new data overlaps existing data at range: "
-                               + str(iset & interval))
-
-        # Insert the data
-        table = self.data.getnode(path)
-        row_start = table.nrows
-        table.append(data)
-        row_end = table.nrows
-
-        # Insert the record into the sql database.
-        self._add_interval(stream_id, interval, row_start, row_end)
-
-        # And that's all
-        return "ok"
-
-    def _find_start(self, table, interval):
-        """
-        Given a DBInterval, find the row in the database that
-        corresponds to the start time.  Return the first database
-        position with a timestamp (first element) greater than or
-        equal to 'start'.
-        """
-        # Optimization for the common case where an interval wasn't truncated
-        if interval.start == interval.db_start:
-            return interval.db_startpos
-        return bisect.bisect_left(bulkdata.TimestampOnlyTable(table),
-                                  interval.start,
-                                  interval.db_startpos,
-                                  interval.db_endpos)
-
-    def _find_end(self, table, interval):
-        """
-        Given a DBInterval, find the row in the database that follows
-        the end time.  Return the first database position after the
-        row with timestamp (first element) greater than or equal
-        to 'end'.
-        """
-        # Optimization for the common case where an interval wasn't truncated
-        if interval.end == interval.db_end:
-            return interval.db_endpos
-        # Note that we still use bisect_left here, because we don't
-        # want to include the given timestamp in the results.  This is
-        # so a queries like 1:00 -> 2:00 and 2:00 -> 3:00 return
-        # non-overlapping data.
-        return bisect.bisect_left(bulkdata.TimestampOnlyTable(table),
-                                  interval.end,
-                                  interval.db_startpos,
-                                  interval.db_endpos)
-
-    def stream_extract(self, path, start = None, end = None, count = False):
-        """
-        Returns (data, restart) tuple.
-
-        data is a list of raw data from the database, suitable for
-        passing to e.g. nilmdb.layout.Formatter to translate into
-        textual form.
-
-        restart, if nonzero, means that there were too many results to
-        return in a single request.  The data is complete from the
-        starting timestamp to the point at which it was truncated,
-        and a new request with a start time of 'restart' will fetch
-        the next block of data.
-
-        count, if true, means to not return raw data, but just the count
-        of rows that would have been returned.  This is much faster
-        than actually fetching the data.  It is not limited by
-        max_results.
-        """
-        table = self.data.getnode(path)
-        stream_id = self._stream_id(path)
-        intervals = self._get_intervals(stream_id)
-        requested = Interval(start or 0, end or 1e12)
-        result = []
-        matched = 0
-        remaining = self.max_results
-        restart = 0
-        for interval in intervals.intersection(requested):
-            # Reading single rows from the table is too slow, so
-            # we use two bisections to find both the starting and
-            # ending row for this particular interval, then
-            # read the entire range as one slice.
-            row_start = self._find_start(table, interval)
-            row_end = self._find_end(table, interval)
-
-            if count:
-                matched += row_end - row_start
-                continue
-
-            # Shorten it if we'll hit the maximum number of results
-            row_max = row_start + remaining
-            if row_max < row_end:
-                row_end = row_max
-                restart = table[row_max][0]
-
-            # Gather these results up
-            result.extend(table[row_start:row_end])
-
-            # Count them
-            remaining -= row_end - row_start
-
-            if restart:
-                break
-
-        if count:
-            return matched
-        return (result, restart)

nilmdb/scripts/__init__.py

@@ -0,0 +1 @@
+# Command line scripts

nilmdb/scripts/nilmdb_server.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python
+
+import nilmdb.server
+import argparse
+import os
+import socket
+
+def main():
+    """Main entry point for the 'nilmdb-server' command line script"""
+
+    parser = argparse.ArgumentParser(
+        description = 'Run the NilmDB server',
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument("-V", "--version", action="version",
+                        version = nilmdb.__version__)
+
+    group = parser.add_argument_group("Standard options")
+    group.add_argument('-a', '--address',
+                       help = 'Only listen on the given address',
+                       default = '0.0.0.0')
+    group.add_argument('-p', '--port', help = 'Listen on the given port',
+                       type = int, default = 12380)
+    group.add_argument('-d', '--database', help = 'Database directory',
+                       default = "./db")
+    group.add_argument('-q', '--quiet', help = 'Silence output',
+                       action = 'store_true')
+    group.add_argument('-t', '--traceback',
+                       help = 'Provide tracebacks in client errors',
+                       action = 'store_true', default = False)
+
+    group = parser.add_argument_group("Debug options")
+    group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and '
+                       'invoke interactive shell afterwards',
+                       action = 'store_true')
+
+    args = parser.parse_args()
+
+    # Create database object.  Needs to be serialized before passing
+    # to the Server.
+    db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database)
+
+    # Configure the server
+    if args.quiet:
+        embedded = True
+    else:
+        embedded = False
+    server = nilmdb.server.Server(db,
+                                  host = args.address,
+                                  port = args.port,
+                                  embedded = embedded,
+                                  force_traceback = args.traceback)
+
+    # Print info
+    if not args.quiet:
+        print "Version: %s" % nilmdb.__version__
+        print "Database: %s" % (os.path.realpath(args.database))
+        if args.address == '0.0.0.0' or args.address == '::':
+            host = socket.getfqdn()
+        else:
+            host = args.address
+        print "Server URL: http://%s:%d/" % ( host, args.port)
+        print "----"
+
+    # Run it
+    if args.yappi:
+        print "Running in yappi"
+        try:
+            import yappi
+            yappi.start()
+            server.start(blocking = True)
+        finally:
+            yappi.stop()
+            yappi.print_stats(sort_type = yappi.SORTTYPE_TTOT, limit = 50)
+            from IPython import embed
+            embed(header = "Use the yappi object to explore further, "
+                  "quit to exit")
+    else:
+        server.start(blocking = True)
+
+    # Clean up
+    if not args.quiet:
+        print "Closing database"
+        db.close()
+
+if __name__ == "__main__":
+    main()

nilmdb/scripts/nilmtool.py

@@ -0,0 +1,10 @@
+#!/usr/bin/python
+
+import nilmdb.cmdline
+
+def main():
+    """Main entry point for the 'nilmtool' command line script"""
+    nilmdb.cmdline.Cmdline().run()
+
+if __name__ == "__main__":
+    main()

nilmdb/server.py

@@ -1,446 +0,0 @@
-"""CherryPy-based server for accessing NILM database via HTTP"""
-
-# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
-# but will pull the nilmdb module instead.
-from __future__ import absolute_import
-from nilmdb.utils.printf import *
-import nilmdb
-
-import cherrypy
-import sys
-import time
-import os
-import simplejson as json
-import functools
-
-try:
-    import cherrypy
-    cherrypy.tools.json_out
-except: # pragma: no cover
-    sys.stderr.write("Cherrypy 3.2+ required\n")
-    sys.exit(1)
-
-class NilmApp(object):
-    def __init__(self, db):
-        self.db = db
-
-version = "1.1"
-
-# Decorators
-def chunked_response(func):
-    """Decorator to enable chunked responses"""
-    # Set this to False to get better tracebacks from some requests
-    # (/stream/extract, /stream/intervals).
-    func._cp_config = { 'response.stream': True }
-    return func
-
-def workaround_cp_bug_1200(func): # pragma: no cover (just a workaround)
-    """Decorator to work around CherryPy bug #1200 in a response
-    generator"""
-    # Even if chunked responses are disabled, you may still miss miss
-    # LookupError, or UnicodeError exceptions due to CherryPy bug
-    # #1200.  This throws them as generic Exceptions insteads.
-    import traceback
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            for val in func(*args, **kwargs):
-                yield val
-        except (LookupError, UnicodeError) as e:
-            raise Exception("bug workaround; real exception is:\n" +
-                            traceback.format_exc())
-    return wrapper
-
-def exception_to_httperror(response = "400 Bad Request"):
-    """Return a decorator that catches Exception and throws
-    a HTTPError describing it instead"""
-    def decorator(func):
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            try:
-                return func(*args, **kwargs)
-            except Exception as e:
-                message = sprintf("%s: %s", type(e).__name__, str(e))
-                raise cherrypy.HTTPError(response, message)
-        return wrapper
-    return decorator
-
-# CherryPy apps
-class Root(NilmApp):
-    """Root application for NILM database"""
-
-    def __init__(self, db, version):
-        super(Root, self).__init__(db)
-        self.server_version = version
-
-    # /
-    @cherrypy.expose
-    def index(self):
-        raise cherrypy.NotFound()
-
-    # /favicon.ico
-    @cherrypy.expose
-    def favicon_ico(self):
-        raise cherrypy.NotFound()
-
-    # /version
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def version(self):
-        return self.server_version
-
-    # /dbpath
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def dbpath(self):
-        return self.db.get_basepath()
-
-    # /dbsize
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def dbsize(self):
-        return nilmdb.utils.du(self.db.get_basepath())
-
-class Stream(NilmApp):
-    """Stream-specific operations"""
-
-    # /stream/list
-    # /stream/list?layout=PrepData
-    # /stream/list?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def list(self, path = None, layout = None):
-        """List all streams in the database.  With optional path or
-        layout parameter, just list streams that match the given path
-        or layout"""
-        return self.db.stream_list(path, layout)
-
-    # /stream/create?path=/newton/prep&layout=PrepData
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror()
-    def create(self, path, layout):
-        """Create a new stream in the database.  Provide path
-        and one of the nilmdb.layout.layouts keys.
-        """
-        return self.db.stream_create(path, layout)
-
-    # /stream/destroy?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror()
-    def destroy(self, path):
-        """Delete a stream and its associated data."""
-        return self.db.stream_destroy(path)
-
-    # /stream/get_metadata?path=/newton/prep
-    # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def get_metadata(self, path, key=None):
-        """Get metadata for the named stream.  If optional
-        key parameters are specified, only return metadata
-        matching the given keys."""
-        try:
-            data = self.db.stream_get_metadata(path)
-        except nilmdb.nilmdb.StreamError as e:
-            raise cherrypy.HTTPError("404 Not Found", e.message)
-        if key is None:  # If no keys specified, return them all
-            key = data.keys()
-        elif not isinstance(key, list):
-            key = [ key ]
-        result = {}
-        for k in key:
-            if k in data:
-                result[k] = data[k]
-            else: # Return "None" for keys with no matching value
-                result[k] = None
-        return result
-
-    # /stream/set_metadata?path=/newton/prep&data=<json>
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror()
-    def set_metadata(self, path, data):
-        """Set metadata for the named stream, replacing any
-        existing metadata.  Data should be a json-encoded
-        dictionary"""
-        data_dict = json.loads(data)
-        self.db.stream_set_metadata(path, data_dict)
-        return "ok"
-
-    # /stream/update_metadata?path=/newton/prep&data=<json>
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror()
-    def update_metadata(self, path, data):
-        """Update metadata for the named stream.  Data
-        should be a json-encoded dictionary"""
-        data_dict = json.loads(data)
-        self.db.stream_update_metadata(path, data_dict)
-        return "ok"
-
-    # /stream/insert?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    #@cherrypy.tools.disable_prb()
-    def insert(self, path, start, end):
-        """
-        Insert new data into the database.  Provide textual data
-        (matching the path's layout) as a HTTP PUT.
-        """
-        # Important that we always read the input before throwing any
-        # errors, to keep lengths happy for persistent connections.
-        # However, CherryPy 3.2.2 has a bug where this fails for GET
-        # requests, so catch that. (issue #1134)
-        try:
-            body = cherrypy.request.body.read()
-        except TypeError:
-            raise cherrypy.HTTPError("400 Bad Request", "No request body")
-
-        # Check path and get layout
-        streams = self.db.stream_list(path = path)
-        if len(streams) != 1:
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
-
-        # Parse the input data
-        try:
-            parser = nilmdb.layout.Parser(layout)
-            parser.parse(body)
-        except nilmdb.layout.ParserError as e:
-            raise cherrypy.HTTPError("400 Bad Request",
-                                     "Error parsing input data: " +
-                                     e.message)
-
-        if (not parser.min_timestamp or not parser.max_timestamp or
-            not len(parser.data)):
-            raise cherrypy.HTTPError("400 Bad Request",
-                                     "no data provided")
-
-        # Check limits
-        start = float(start)
-        end = float(end)
-        if parser.min_timestamp < start:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.min_timestamp) +
-                                     " < start time " + repr(start))
-        if parser.max_timestamp >= end:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.max_timestamp) +
-                                     " >= end time " + repr(end))
-
-        # Now do the nilmdb insert, passing it the parser full of data.
-        try:
-            result = self.db.stream_insert(path, start, end, parser.data)
-        except nilmdb.nilmdb.NilmDBError as e:
-            raise cherrypy.HTTPError("400 Bad Request", e.message)
-
-        # Done
-        return "ok"
-
-    # /stream/intervals?path=/newton/prep
-    # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    @cherrypy.expose
-    @chunked_response
-    def intervals(self, path, start = None, end = None):
-        """
-        Get intervals from backend database.  Streams the resulting
-        intervals as JSON strings separated by newlines.  This may
-        make multiple requests to the nilmdb backend to avoid causing
-        it to block for too long.
-        """
-        if start is not None:
-            start = float(start)
-        if end is not None:
-            end = float(end)
-
-        if start is not None and end is not None:
-            if end < start:
-                raise cherrypy.HTTPError("400 Bad Request",
-                                         "end before start")
-
-        streams = self.db.stream_list(path = path)
-        if len(streams) != 1:
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-
-        @workaround_cp_bug_1200
-        def content(start, end):
-            # Note: disable chunked responses to see tracebacks from here.
-            while True:
-                (intervals, restart) = self.db.stream_intervals(path,start,end)
-                response = ''.join([ json.dumps(i) + "\n" for i in intervals ])
-                yield response
-                if restart == 0:
-                    break
-                start = restart
-        return content(start, end)
-
-    # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    @cherrypy.expose
-    @chunked_response
-    def extract(self, path, start = None, end = None, count = False):
-        """
-        Extract data from backend database.  Streams the resulting
-        entries as ASCII text lines separated by newlines.  This may
-        make multiple requests to the nilmdb backend to avoid causing
-        it to block for too long.
-
-        Add count=True to return a count rather than actual data.
-        """
-        if start is not None:
-            start = float(start)
-        if end is not None:
-            end = float(end)
-
-        # Check parameters
-        if start is not None and end is not None:
-            if end < start:
-                raise cherrypy.HTTPError("400 Bad Request",
-                                         "end before start")
-
-        # Check path and get layout
-        streams = self.db.stream_list(path = path)
-        if len(streams) != 1:
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
-
-        # Get formatter
-        formatter = nilmdb.layout.Formatter(layout)
-
-        @workaround_cp_bug_1200
-        def content(start, end, count):
-            # Note: disable chunked responses to see tracebacks from here.
-            if count:
-                matched = self.db.stream_extract(path, start, end, count)
-                yield sprintf("%d\n", matched)
-                return
-
-            while True:
-                (data, restart) = self.db.stream_extract(path, start, end)
-
-                # Format the data and yield it
-                yield formatter.format(data)
-
-                if restart == 0:
-                    return
-                start = restart
-        return content(start, end, count)
-
-class Exiter(object):
-    """App that exits the server, for testing"""
-    @cherrypy.expose
-    def index(self):
-        cherrypy.response.headers['Content-Type'] = 'text/plain'
-        def content():
-            yield 'Exiting by request'
-            raise SystemExit
-        return content()
-    index._cp_config = { 'response.stream': True }
-
-class Server(object):
-    def __init__(self, db, host = '127.0.0.1', port = 8080,
-                 stoppable = False,       # whether /exit URL exists
-                 embedded = True,         # hide diagnostics and output, etc
-                 fast_shutdown = False,   # don't wait for clients to disconn.
-                 force_traceback = False  # include traceback in all errors
-                 ):
-        self.version = version
-
-        # Need to wrap DB object in a serializer because we'll call
-        # into it from separate threads.
-        self.embedded = embedded
-        self.db = nilmdb.utils.Serializer(db)
-        cherrypy.config.update({
-            'server.socket_host': host,
-            'server.socket_port': port,
-            'engine.autoreload_on': False,
-            'server.max_request_body_size': 4*1024*1024,
-            'error_page.default': self.json_error_page,
-            })
-        if self.embedded:
-            cherrypy.config.update({ 'environment': 'embedded' })
-
-        # Send tracebacks in error responses.  They're hidden by the
-        # error_page function for client errors (code 400-499).
-        cherrypy.config.update({ 'request.show_tracebacks' : True })
-        self.force_traceback = force_traceback
-
-        cherrypy.tree.apps = {}
-        cherrypy.tree.mount(Root(self.db, self.version), "/")
-        cherrypy.tree.mount(Stream(self.db), "/stream")
-        if stoppable:
-            cherrypy.tree.mount(Exiter(), "/exit")
-
-        # Shutdowns normally wait for clients to disconnect.  To speed
-        # up tests, set fast_shutdown = True
-        if fast_shutdown:
-            # Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
-            cherrypy.server.shutdown_timeout = 0.01
-        else:
-            cherrypy.server.shutdown_timeout = 5
-
-    def json_error_page(self, status, message, traceback, version):
-        """Return a custom error page in JSON so the client can parse it"""
-        errordata = { "status" : status,
-                      "message" : message,
-                      "traceback" : traceback }
-        # Don't send a traceback if the error was 400-499 (client's fault)
-        try:
-            code = int(status.split()[0])
-            if not self.force_traceback:
-                if code >= 400 and code <= 499:
-                    errordata["traceback"] = ""
-        except Exception as e: # pragma: no cover
-            pass
-        # Override the response type, which was previously set to text/html
-        cherrypy.serving.response.headers['Content-Type'] = (
-            "application/json;charset=utf-8" )
-        # Undo the HTML escaping that cherrypy's get_error_page function applies
-        # (cherrypy issue 1135)
-        for k, v in errordata.iteritems():
-            v = v.replace("&lt;","<")
-            v = v.replace("&gt;",">")
-            v = v.replace("&amp;","&")
-            errordata[k] = v
-        return json.dumps(errordata, separators=(',',':'))
-
-    def start(self, blocking = False, event = None):
-
-        if not self.embedded: # pragma: no cover
-            # Handle signals nicely
-            if hasattr(cherrypy.engine, "signal_handler"):
-                cherrypy.engine.signal_handler.subscribe()
-            if hasattr(cherrypy.engine, "console_control_handler"):
-                cherrypy.engine.console_control_handler.subscribe()
-
-        # Cherrypy stupidly calls os._exit(70) when it can't bind the
-        # port.  At least try to print a reasonable error and continue
-        # in this case, rather than just dying silently (as we would
-        # otherwise do in embedded mode)
-        real_exit = os._exit
-        def fake_exit(code): # pragma: no cover
-            if code == os.EX_SOFTWARE:
-                fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
-            else:
-                real_exit(code)
-        os._exit = fake_exit
-        cherrypy.engine.start()
-        os._exit = real_exit
-
-        if event is not None:
-            event.set()
-        if blocking:
-            try:
-                cherrypy.engine.wait(cherrypy.engine.states.EXITING,
-                                     interval = 0.1, channel = 'main')
-            except (KeyboardInterrupt, IOError): # pragma: no cover
-                cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
-                cherrypy.engine.exit()
-            except SystemExit: # pragma: no cover
-                cherrypy.engine.log('SystemExit raised: shutting down bus')
-                cherrypy.engine.exit()
-                raise
-
-    def stop(self):
-        cherrypy.engine.exit()

nilmdb/server/__init__.py

@@ -0,0 +1,21 @@
+"""nilmdb.server"""
+
+from __future__ import absolute_import
+
+# Try to set up pyximport to automatically rebuild Cython modules.  If
+# this doesn't work, it's OK, as long as the modules were built externally.
+# (e.g. python setup.py build_ext --inplace)
+try: # pragma: no cover
+    import Cython
+    import distutils.version
+    if (distutils.version.LooseVersion(Cython.__version__) <
+        distutils.version.LooseVersion("0.17")): # pragma: no cover
+        raise ImportError("Cython version too old")
+    import pyximport
+    pyximport.install(inplace = True, build_in_temp = False)
+except (ImportError, TypeError): # pragma: no cover
+    pass
+
+from nilmdb.server.nilmdb import NilmDB
+from nilmdb.server.server import Server, wsgi_application
+from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

nilmdb/server/bulkdata.py

@@ -0,0 +1,594 @@
+# Fixed record size bulk data storage
+
+# Need absolute_import so that "import nilmdb" won't pull in
+# nilmdb.py, but will pull the parent nilmdb module instead.
+from __future__ import absolute_import
+from __future__ import division
+from nilmdb.utils.printf import *
+from nilmdb.utils.time import timestamp_to_string as timestamp_to_string
+import nilmdb.utils
+
+import os
+import cPickle as pickle
+import re
+import sys
+import tempfile
+
+import nilmdb.utils.lock
+from . import rocket
+
+# Up to 256 open file descriptors at any given time.
+# These variables are global so they can be used in the decorator arguments.
+table_cache_size = 16
+fd_cache_size = 16
+
+@nilmdb.utils.must_close(wrap_verify = False)
+class BulkData(object):
+    def __init__(self, basepath, **kwargs):
+        self.basepath = basepath
+        self.root = os.path.join(self.basepath, "data")
+        self.lock = self.root + ".lock"
+        self.lockfile = None
+
+        # Tuneables
+        if "file_size" in kwargs:
+            self.file_size = kwargs["file_size"]
+        else:
+            # Default to approximately 128 MiB per file
+            self.file_size = 128 * 1024 * 1024
+
+        if "files_per_dir" in kwargs:
+            self.files_per_dir = kwargs["files_per_dir"]
+        else:
+            # 32768 files per dir should work even on FAT32
+            self.files_per_dir = 32768
+
+        # Make root path
+        if not os.path.isdir(self.root):
+            os.mkdir(self.root)
+
+        # Create the lock
+        self.lockfile = open(self.lock, "w")
+        if not nilmdb.utils.lock.exclusive_lock(self.lockfile):
+            raise IOError('database at "' + self.basepath +
+                          '" is already locked by another process')
+
+    def close(self):
+        self.getnode.cache_remove_all()
+        if self.lockfile:
+            nilmdb.utils.lock.exclusive_unlock(self.lockfile)
+            self.lockfile.close()
+            try:
+                os.unlink(self.lock)
+            except OSError: # pragma: no cover
+                pass
+            self.lockfile = None
+
+    def _encode_filename(self, path):
+        # Encode all paths to UTF-8, regardless of sys.getfilesystemencoding(),
+        # because we want to be able to represent all code points and the user
+        # will never be directly exposed to filenames.  We can then do path
+        # manipulations on the UTF-8 directly.
+        if isinstance(path, unicode):
+            return path.encode('utf-8')
+        return path
+
+    def _create_check_ospath(self, ospath):
+        if ospath[-1] == '/':
+            raise ValueError("invalid path; should not end with a /")
+        if Table.exists(ospath):
+            raise ValueError("stream already exists at this path")
+        if os.path.isdir(ospath):
+            raise ValueError("subdirs of this path already exist")
+
+    def _create_parents(self, unicodepath):
+        """Verify the path name, and create parent directories if they
+        don't exist.  Returns a list of elements that got created."""
+        path = self._encode_filename(unicodepath)
+
+        if path[0] != '/':
+            raise ValueError("paths must start with /")
+        [ group, node ] = path.rsplit("/", 1)
+        if group == '':
+            raise ValueError("invalid path; path must contain at least one "
+                             "folder")
+        if node == '':
+            raise ValueError("invalid path; should not end with a /")
+        if not Table.valid_path(path):
+            raise ValueError("path name is invalid or contains reserved words")
+
+        # Create the table's base dir.  Note that we make a
+        # distinction here between NilmDB paths (always Unix style,
+        # split apart manually) and OS paths (built up with
+        # os.path.join)
+
+        # Make directories leading up to this one
+        elements = path.lstrip('/').split('/')
+        made_dirs = []
+        try:
+            # Make parent elements
+            for i in range(len(elements)):
+                ospath = os.path.join(self.root, *elements[0:i])
+                if Table.exists(ospath):
+                    raise ValueError("path is subdir of existing node")
+                if not os.path.isdir(ospath):
+                    os.mkdir(ospath)
+                    made_dirs.append(ospath)
+        except Exception as e:
+            # Try to remove paths that we created; ignore errors
+            exc_info = sys.exc_info()
+            for ospath in reversed(made_dirs): # pragma: no cover (hard to hit)
+                try:
+                    os.rmdir(ospath)
+                except OSError:
+                    pass
+            raise exc_info[1], None, exc_info[2]
+
+        return elements
+
+    def create(self, unicodepath, layout_name):
+        """
+        unicodepath: path to the data (e.g. u'/newton/prep').
+        Paths must contain at least two elements, e.g.:
+           /newton/prep
+           /newton/raw
+           /newton/upstairs/prep
+           /newton/upstairs/raw
+
+        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
+        """
+        elements = self._create_parents(unicodepath)
+
+        # Make the final dir
+        ospath = os.path.join(self.root, *elements)
+        self._create_check_ospath(ospath)
+        os.mkdir(ospath)
+
+        try:
+            # Write format string to file
+            Table.create(ospath, layout_name, self.file_size,
+                         self.files_per_dir)
+
+            # Open and cache it
+            self.getnode(unicodepath)
+        except Exception:
+            exc_info = sys.exc_info()
+            try:
+                os.rmdir(ospath)
+            except OSError:
+                pass
+            raise exc_info[1], None, exc_info[2]
+
+        # Success
+        return
+
+    def _remove_leaves(self, unicodepath):
+        """Remove empty directories starting at the leaves of unicodepath"""
+        path = self._encode_filename(unicodepath)
+        elements = path.lstrip('/').split('/')
+        for i in reversed(range(len(elements))):
+            ospath = os.path.join(self.root, *elements[0:i+1])
+            try:
+                os.rmdir(ospath)
+            except OSError:
+                pass
+
+    def rename(self, oldunicodepath, newunicodepath):
+        """Move entire tree from 'oldunicodepath' to
+        'newunicodepath'"""
+        oldpath = self._encode_filename(oldunicodepath)
+        newpath = self._encode_filename(newunicodepath)
+
+        # Get OS paths
+        oldelements = oldpath.lstrip('/').split('/')
+        oldospath = os.path.join(self.root, *oldelements)
+        newelements = newpath.lstrip('/').split('/')
+        newospath = os.path.join(self.root, *newelements)
+
+        # Basic checks
+        if oldospath == newospath:
+            raise ValueError("old and new paths are the same")
+        self._create_check_ospath(newospath)
+
+        # Move the table to a temporary location
+        tmpdir = tempfile.mkdtemp(prefix = "rename-", dir = self.root)
+        tmppath = os.path.join(tmpdir, "table")
+        os.rename(oldospath, tmppath)
+
+        try:
+            # Create parent dirs for new location
+            self._create_parents(newunicodepath)
+
+            # Move table into new location
+            os.rename(tmppath, newospath)
+        except Exception:
+            # On failure, move the table back to original path
+            os.rename(tmppath, oldospath)
+            os.rmdir(tmpdir)
+            raise
+
+        # Prune old dirs
+        self._remove_leaves(oldunicodepath)
+        os.rmdir(tmpdir)
+
+    def destroy(self, unicodepath):
+        """Fully remove all data at a particular path.  No way to undo
+        it!  The group/path structure is removed, too."""
+        path = self._encode_filename(unicodepath)
+
+        # Get OS path
+        elements = path.lstrip('/').split('/')
+        ospath = os.path.join(self.root, *elements)
+
+        # Remove Table object from cache
+        self.getnode.cache_remove(self, unicodepath)
+
+        # Remove the contents of the target directory
+        if not Table.exists(ospath):
+            raise ValueError("nothing at that path")
+        for (root, dirs, files) in os.walk(ospath, topdown = False):
+            for name in files:
+                os.remove(os.path.join(root, name))
+            for name in dirs:
+                os.rmdir(os.path.join(root, name))
+
+        # Remove leftover empty directories
+        self._remove_leaves(unicodepath)
+
+    # Cache open tables
+    @nilmdb.utils.lru_cache(size = table_cache_size,
+                            onremove = lambda x: x.close())
+    def getnode(self, unicodepath):
+        """Return a Table object corresponding to the given database
+        path, which must exist."""
+        path = self._encode_filename(unicodepath)
+        elements = path.lstrip('/').split('/')
+        ospath = os.path.join(self.root, *elements)
+        return Table(ospath)
+
+@nilmdb.utils.must_close(wrap_verify = False)
+class Table(object):
+    """Tools to help access a single table (data at a specific OS path)."""
+    # See design.md for design details
+
+    # Class methods, to help keep format details in this class.
+    @classmethod
+    def valid_path(cls, root):
+        """Return True if a root path is a valid name"""
+        return "_format" not in root.split("/")
+
+    @classmethod
+    def exists(cls, root):
+        """Return True if a table appears to exist at this OS path"""
+        return os.path.isfile(os.path.join(root, "_format"))
+
+    @classmethod
+    def create(cls, root, layout, file_size, files_per_dir):
+        """Initialize a table at the given OS path with the
+        given layout string"""
+
+        # Calculate rows per file so that each file is approximately
+        # file_size bytes.
+        rkt = rocket.Rocket(layout, None)
+        rows_per_file = max(file_size // rkt.binary_size, 1)
+        rkt.close()
+
+        fmt = { "rows_per_file": rows_per_file,
+                "files_per_dir": files_per_dir,
+                "layout": layout,
+                "version": 3 }
+        with open(os.path.join(root, "_format"), "wb") as f:
+            pickle.dump(fmt, f, 2)
+
+    # Normal methods
+    def __init__(self, root):
+        """'root' is the full OS path to the directory of this table"""
+        self.root = root
+
+        # Load the format
+        with open(os.path.join(self.root, "_format"), "rb") as f:
+            fmt = pickle.load(f)
+
+        if fmt["version"] != 3: # pragma: no cover
+            # Old versions used floating point timestamps, which aren't
+            # valid anymore.
+            raise NotImplementedError("old version " + str(fmt["version"]) +
+                                      " bulk data store is not supported")
+
+        self.rows_per_file = fmt["rows_per_file"]
+        self.files_per_dir = fmt["files_per_dir"]
+        self.layout = fmt["layout"]
+
+        # Use rocket to get row size and file size
+        rkt = rocket.Rocket(self.layout, None)
+        self.row_size = rkt.binary_size
+        self.file_size = rkt.binary_size * self.rows_per_file
+        rkt.close()
+
+        # Find nrows
+        self.nrows = self._get_nrows()
+
+    def close(self):
+        self.file_open.cache_remove_all()
+
+    # Internal helpers
+    def _get_nrows(self):
+        """Find nrows by locating the lexicographically last filename
+        and using its size"""
+        # Note that this just finds a 'nrows' that is guaranteed to be
+        # greater than the row number of any piece of data that
+        # currently exists, not necessarily all data that _ever_
+        # existed.
+        regex = re.compile("^[0-9a-f]{4,}$")
+
+        # Find the last directory.  We sort and loop through all of them,
+        # starting with the numerically greatest, because the dirs could be
+        # empty if something was deleted.
+        subdirs = sorted(filter(regex.search, os.listdir(self.root)),
+                         key = lambda x: int(x, 16), reverse = True)
+
+        for subdir in subdirs:
+            # Now find the last file in that dir
+            path = os.path.join(self.root, subdir)
+            files = filter(regex.search, os.listdir(path))
+            if not files: # pragma: no cover (shouldn't occur)
+                # Empty dir: try the next one
+                continue
+
+            # Find the numerical max
+            filename = max(files, key = lambda x: int(x, 16))
+            offset = os.path.getsize(os.path.join(self.root, subdir, filename))
+
+            # Convert to row number
+            return self._row_from_offset(subdir, filename, offset)
+
+        # No files, so no data
+        return 0
+
+    def _offset_from_row(self, row):
+        """Return a (subdir, filename, offset, count) tuple:
+
+          subdir: subdirectory for the file
+        filename: the filename that contains the specified row
+          offset: byte offset of the specified row within the file
+           count: number of rows (starting at offset) that fit in the file
+        """
+        filenum = row // self.rows_per_file
+        # It's OK if these format specifiers are too short; the filenames
+        # will just get longer but will still sort correctly.
+        dirname = sprintf("%04x", filenum // self.files_per_dir)
+        filename = sprintf("%04x", filenum % self.files_per_dir)
+        offset = (row % self.rows_per_file) * self.row_size
+        count = self.rows_per_file - (row % self.rows_per_file)
+        return (dirname, filename, offset, count)
+
+    def _row_from_offset(self, subdir, filename, offset):
+        """Return the row number that corresponds to the given
+        'subdir/filename' and byte-offset within that file."""
+        if (offset % self.row_size) != 0: # pragma: no cover
+            # this shouldn't occur, unless there is some corruption somewhere
+            raise ValueError("file offset is not a multiple of data size")
+        filenum = int(subdir, 16) * self.files_per_dir + int(filename, 16)
+        row = (filenum * self.rows_per_file) + (offset // self.row_size)
+        return row
+
+    def _remove_or_truncate_file(self, subdir, filename, offset = 0):
+        """Remove the given file, and remove the subdirectory too
+        if it's empty.  If offset is nonzero, truncate the file
+        to that size instead."""
+        # Close potentially open file in file_open LRU cache
+        self.file_open.cache_remove(self, subdir, filename)
+        if offset:
+            # Truncate it
+            with open(os.path.join(self.root, subdir, filename), "r+b") as f:
+                f.truncate(offset)
+        else:
+            # Remove file
+            os.remove(os.path.join(self.root, subdir, filename))
+            # Try deleting subdir, too
+            try:
+                os.rmdir(os.path.join(self.root, subdir))
+            except Exception:
+                pass
+
+    # Cache open files
+    @nilmdb.utils.lru_cache(size = fd_cache_size,
+                            onremove = lambda f: f.close())
+    def file_open(self, subdir, filename):
+        """Open and map a given 'subdir/filename' (relative to self.root).
+        Will be automatically closed when evicted from the cache."""
+        # Create path if it doesn't exist
+        try:
+            os.mkdir(os.path.join(self.root, subdir))
+        except OSError:
+            pass
+        # Return a rocket.Rocket object, which contains the open file
+        return rocket.Rocket(self.layout,
+                             os.path.join(self.root, subdir, filename))
+
+    def append_string(self, data, start, end):
+        """Parse the formatted string in 'data', according to the
+        current layout, and append it to the table.  If any timestamps
+        are non-monotonic, or don't fall between 'start' and 'end',
+        a ValueError is raised.
+
+        If this function succeeds, it returns normally.  Otherwise,
+        the table is reverted back to its original state by truncating
+        or deleting files as necessary."""
+        data_offset = 0
+        last_timestamp = nilmdb.utils.time.min_timestamp
+        tot_rows = self.nrows
+        count = 0
+        linenum = 0
+        try:
+            while data_offset < len(data):
+                # See how many rows we can fit into the current file,
+                # and open it
+                (subdir, fname, offset, count) = self._offset_from_row(tot_rows)
+                f = self.file_open(subdir, fname)
+
+                # Ask the rocket object to parse and append up to "count"
+                # rows of data, verifying things along the way.
+                try:
+                    (added_rows, data_offset, last_timestamp, linenum
+                     ) = f.append_string(count, data, data_offset, linenum,
+                                         start, end, last_timestamp)
+                except rocket.ParseError as e:
+                    (linenum, colnum, errtype, obj) = e.args
+                    where = "line %d, column %d: " % (linenum, colnum)
+                    # Extract out the error line, add column marker
+                    try:
+                        bad = data.splitlines()[linenum-1]
+                        badptr = ' ' * (colnum - 1) + '^'
+                    except IndexError: # pragma: no cover
+                        bad = ""
+                    if errtype == rocket.ERR_NON_MONOTONIC:
+                        err = "timestamp is not monotonically increasing"
+                    elif errtype == rocket.ERR_OUT_OF_INTERVAL:
+                        if obj < start:
+                            err = sprintf("Data timestamp %s < start time %s",
+                                          timestamp_to_string(obj),
+                                          timestamp_to_string(start))
+                        else:
+                            err = sprintf("Data timestamp %s >= end time %s",
+                                          timestamp_to_string(obj),
+                                          timestamp_to_string(end))
+                    else:
+                        err = str(obj)
+                    raise ValueError("error parsing input data: " +
+                                     where + err + "\n" + bad + "\n" + badptr)
+                tot_rows += added_rows
+        except Exception:
+            # Some failure, so try to roll things back by truncating or
+            # deleting files that we may have appended data to.
+            cleanpos = self.nrows
+            while cleanpos <= tot_rows:
+                (subdir, fname, offset, count) = self._offset_from_row(cleanpos)
+                self._remove_or_truncate_file(subdir, fname, offset)
+                cleanpos += count
+            # Re-raise original exception
+            raise
+        else:
+            # Success, so update self.nrows accordingly
+            self.nrows = tot_rows
+
+    def get_data(self, start, stop):
+        """Extract data corresponding to Python range [n:m],
+        and returns a formatted string"""
+        if (start is None or
+            stop is None or
+            start > stop or
+            start < 0 or
+            stop > self.nrows):
+            raise IndexError("Index out of range")
+
+        ret = []
+        row = start
+        remaining = stop - start
+        while remaining > 0:
+            (subdir, filename, offset, count) = self._offset_from_row(row)
+            if count > remaining:
+                count = remaining
+            f = self.file_open(subdir, filename)
+            ret.append(f.extract_string(offset, count))
+            remaining -= count
+            row += count
+        return "".join(ret)
+
+    def __getitem__(self, row):
+        """Extract timestamps from a row, with table[n] notation."""
+        if row < 0 or row >= self.nrows:
+            raise IndexError("Index out of range")
+        (subdir, filename, offset, count) = self._offset_from_row(row)
+        f = self.file_open(subdir, filename)
+        return f.extract_timestamp(offset)
+
+    def _remove_rows(self, subdir, filename, start, stop):
+        """Helper to mark specific rows as being removed from a
+        file, and potentially remove or truncate the file itself."""
+        # Close potentially open file in file_open LRU cache
+        self.file_open.cache_remove(self, subdir, filename)
+
+        # We keep a file like 0000.removed that contains a list of
+        # which rows have been "removed".  Note that we never have to
+        # remove entries from this list, because we never decrease
+        # self.nrows, and so we will never overwrite those locations in the
+        # file.  Only when the list covers the entire extent of the
+        # file will that file be removed.
+        datafile = os.path.join(self.root, subdir, filename)
+        cachefile = datafile + ".removed"
+        try:
+            with open(cachefile, "rb") as f:
+                ranges = pickle.load(f)
+            cachefile_present = True
+        except Exception:
+            ranges = []
+            cachefile_present = False
+
+        # Append our new range and sort
+        ranges.append((start, stop))
+        ranges.sort()
+
+        # Merge adjacent ranges into "out"
+        merged = []
+        prev = None
+        for new in ranges:
+            if prev is None:
+                # No previous range, so remember this one
+                prev = new
+            elif prev[1] == new[0]:
+                # Previous range connected to this new one; extend prev
+                prev = (prev[0], new[1])
+            else:
+                # Not connected; append previous and start again
+                merged.append(prev)
+                prev = new
+        if prev is not None:
+            merged.append(prev)
+
+        # If the range covered the whole file, we can delete it now.
+        # Note that the last file in a table may be only partially
+        # full (smaller than self.rows_per_file).  We purposely leave
+        # those files around rather than deleting them, because the
+        # remainder will be filled on a subsequent append(), and things
+        # are generally easier if we don't have to special-case that.
+        if (len(merged) == 1 and
+            merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
+            # Delete files
+            if cachefile_present:
+                os.remove(cachefile)
+            self._remove_or_truncate_file(subdir, filename, 0)
+        else:
+            # File needs to stick around.  This means we can get
+            # degenerate cases where we have large files containing as
+            # little as one row.  Try to punch a hole in the file,
+            # so that this region doesn't take up filesystem space.
+            offset = start * self.row_size
+            count = (stop - start) * self.row_size
+            nilmdb.utils.fallocate.punch_hole(datafile, offset, count)
+
+            # Update cache.  Try to do it atomically.
+            nilmdb.utils.atomic.replace_file(cachefile,
+                                             pickle.dumps(merged, 2))
+
+    def remove(self, start, stop):
+        """Remove specified rows [start, stop) from this table.
+
+        If a file is left empty, it is fully removed.  Otherwise, a
+        parallel data file is used to remember which rows have been
+        removed, and the file is otherwise untouched."""
+        if start < 0 or start > stop or stop > self.nrows:
+            raise IndexError("Index out of range")
+
+        row = start
+        remaining = stop - start
+        while remaining:
+            # Loop through each file that we need to touch
+            (subdir, filename, offset, count) = self._offset_from_row(row)
+            if count > remaining:
+                count = remaining
+            row_offset = offset // self.row_size
+            # Mark the rows as being removed
+            self._remove_rows(subdir, filename, row_offset, row_offset + count)
+            remaining -= count
+            row += count

nilmdb/server/errors.py

@@ -0,0 +1,12 @@
+"""Exceptions"""
+
+class NilmDBError(Exception):
+    """Base exception for NilmDB errors"""
+    def __init__(self, message = "Unspecified error"):
+        Exception.__init__(self, message)
+
+class StreamError(NilmDBError):
+    pass
+
+class OverlapError(NilmDBError):
+    pass

nilmdb/server/interval.pyx

@@ -1,5 +1,9 @@
 """Interval, IntervalSet
 
+The Interval implemented here is just like
+nilmdb.utils.interval.Interval, except implemented in Cython for
+speed.
+
 Represents an interval of time, and a set of such intervals.
 
 Intervals are half-open, ie. they include data points with timestamps
@@ -19,48 +23,44 @@ Intervals are half-open, ie. they include data points with timestamps
 # Fourth version is an optimized rb-tree that stores interval starts
 # and ends directly in the tree, like bxinterval did.
 
-cimport rbtree
-cdef extern from "stdint.h":
-    ctypedef unsigned long long uint64_t
+from ..utils.time import min_timestamp as nilmdb_min_timestamp
+from ..utils.time import max_timestamp as nilmdb_max_timestamp
+from ..utils.time import timestamp_to_string
+from ..utils.iterator import imerge
+from ..utils.interval import IntervalError
+import itertools
 
-class IntervalError(Exception):
-    """Error due to interval overlap, etc"""
-    pass
+cimport rbtree
+from libc.stdint cimport uint64_t, int64_t
+
+ctypedef int64_t timestamp_t
 
 cdef class Interval:
     """Represents an interval of time."""
 
-    cdef public double start, end
+    cdef public timestamp_t start, end
 
-    def __init__(self, double start, double end):
+    def __init__(self, timestamp_t start, timestamp_t end):
         """
-        'start' and 'end' are arbitrary floats that represent time
+        'start' and 'end' are arbitrary numbers that represent time
         """
-        if start > end:
+        if start >= end:
+            # Explicitly disallow zero-width intervals (since they're half-open)
             raise IntervalError("start %s must precede end %s" % (start, end))
-        self.start = float(start)
-        self.end = float(end)
+        self.start = start
+        self.end = end
 
     def __repr__(self):
         s = repr(self.start) + ", " + repr(self.end)
         return self.__class__.__name__ + "(" + s + ")"
 
     def __str__(self):
-        return "[" + repr(self.start) + " -> " + repr(self.end) + ")"
+        return ("[" + timestamp_to_string(self.start) +
+                " -> " + timestamp_to_string(self.end) + ")")
 
     def __cmp__(self, Interval other):
         """Compare two intervals.  If non-equal, order by start then end"""
-        if not isinstance(other, Interval):
-            raise TypeError("bad type")
-        if self.start == other.start:
-            if self.end < other.end:
-                return -1
-            if self.end > other.end:
-                return 1
-            return 0
-        if self.start < other.start:
-            return -1
-        return 1
+        return cmp(self.start, other.start) or cmp(self.end, other.end)
 
     cpdef intersects(self, Interval other):
         """Return True if two Interval objects intersect"""
@@ -68,7 +68,7 @@ cdef class Interval:
             return False
         return True
 
-    cpdef subset(self, double start, double end):
+    cpdef subset(self, timestamp_t start, timestamp_t end):
         """Return a new Interval that is a subset of this one"""
         # A subclass that tracks additional data might override this.
         if start < self.start or end > self.end:
@@ -90,14 +90,14 @@ cdef class DBInterval(Interval):
         db_end = 200, db_endpos = 20000
     """
 
-    cpdef public double db_start, db_end
+    cpdef public timestamp_t db_start, db_end
     cpdef public uint64_t db_startpos, db_endpos
 
     def __init__(self, start, end,
                  db_start, db_end,
                  db_startpos, db_endpos):
         """
-        'db_start' and 'db_end' are arbitrary floats that represent
+        'db_start' and 'db_end' are arbitrary numbers that represent
         time.  They must be a strict superset of the time interval
         covered by 'start' and 'end'.  The 'db_startpos' and
         'db_endpos' are arbitrary database position indicators that
@@ -117,7 +117,7 @@ cdef class DBInterval(Interval):
         s += ", " + repr(self.db_startpos) + ", " + repr(self.db_endpos)
         return self.__class__.__name__ + "(" + s + ")"
 
-    cpdef subset(self, double start, double end):
+    cpdef subset(self, timestamp_t start, timestamp_t end):
         """
         Return a new DBInterval that is a subset of this one
         """
@@ -261,24 +261,18 @@ cdef class IntervalSet:
 
     def __and__(self, other not None):
         """
-        Compute a new IntervalSet from the intersection of two others
+        Compute a new IntervalSet from the intersection of this
+        IntervalSet with one other interval.
 
         Output intervals are built as subsets of the intervals in the
         first argument (self).
         """
         out = IntervalSet()
-
-        if not isinstance(other, IntervalSet):
         for i in self.intersection(other):
             out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-        else:
-            for x in other:
-                for i in self.intersection(x):
-                    out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-
         return out
 
-    def intersection(self, Interval interval not None):
+    def intersection(self, Interval interval not None, orig = False):
         """
         Compute a sequence of intervals that correspond to the
         intersection between `self` and the provided interval.
@@ -287,15 +281,20 @@ cdef class IntervalSet:
 
         Output intervals are built as subsets of the intervals in the
         first argument (self).
+
+        If orig = True, also return the original interval that was
+        (potentially) subsetted to make the one that is being
+        returned.
         """
-        if not isinstance(interval, Interval):
-            raise TypeError("bad type")
+        if orig:
             for n in self.tree.intersect(interval.start, interval.end):
                 i = n.obj
-            if i:
-                if i.start >= interval.start and i.end <= interval.end:
-                    yield i
+                subset = i.subset(max(i.start, interval.start),
+                                  min(i.end, interval.end))
+                yield (subset, i)
         else:
+            for n in self.tree.intersect(interval.start, interval.end):
+                i = n.obj
                 subset = i.subset(max(i.start, interval.start),
                                   min(i.end, interval.end))
                 yield subset
@@ -307,7 +306,7 @@ cdef class IntervalSet:
                 return True
         return False
 
-    def find_end(self, double t):
+    def find_end(self, timestamp_t t):
         """
         Return an Interval from this tree that ends at time t, or
         None if it doesn't exist.

nilmdb/server/nilmdb.py

@@ -0,0 +1,673 @@
+# -*- coding: utf-8 -*-
+
+"""NilmDB
+
+Object that represents a NILM database file.
+
+Manages both the SQL database and the table storage backend.
+"""
+
+# Need absolute_import so that "import nilmdb" won't pull in
+# nilmdb.py, but will pull the parent nilmdb module instead.
+from __future__ import absolute_import
+import nilmdb.utils
+from nilmdb.utils.printf import *
+from nilmdb.utils.time import timestamp_to_string
+
+from nilmdb.utils.interval import IntervalError
+from nilmdb.server.interval import Interval, DBInterval, IntervalSet
+
+from nilmdb.server import bulkdata
+from nilmdb.server.errors import NilmDBError, StreamError, OverlapError
+
+import sqlite3
+import os
+import errno
+import bisect
+
+# Note about performance and transactions:
+#
+# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
+# takes about 125msec.  sqlite3 will commit transactions at 3 times:
+# 1: explicit con.commit()
+# 2: between a series of DML commands and non-DML commands, e.g.
+#    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
+# 3: at the end of an explicit transaction, e.g. "with self.con as con:"
+#
+# To speed things up, we can set 'PRAGMA synchronous=OFF'.  Or, it
+# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL'
+# give an equivalent speedup more safely.  That is what is used here.
+_sql_schema_updates = {
+    0: { "next": 1, "sql": """
+    -- All streams
+    CREATE TABLE streams(
+        id INTEGER PRIMARY KEY,		-- stream ID
+        path TEXT UNIQUE NOT NULL,	-- path, e.g. '/newton/prep'
+        layout TEXT NOT NULL		-- layout name, e.g. float32_8
+    );
+
+    -- Individual timestamped ranges in those streams.
+    -- For a given start_time and end_time, this tells us that the
+    -- data is stored between start_pos and end_pos.
+    -- Times are stored as μs since Unix epoch
+    -- Positions are opaque: PyTables rows, file offsets, etc.
+    --
+    -- Note: end_pos points to the row _after_ end_time, so end_pos-1
+    -- is the last valid row.
+    CREATE TABLE ranges(
+        stream_id INTEGER NOT NULL,
+        start_time INTEGER NOT NULL,
+        end_time INTEGER NOT NULL,
+        start_pos INTEGER NOT NULL,
+        end_pos INTEGER NOT NULL
+    );
+    CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time);
+    """ },
+
+    1: { "next": 3, "sql": """
+    -- Generic dictionary-type metadata that can be associated with a stream
+    CREATE TABLE metadata(
+    	stream_id INTEGER NOT NULL,
+        key TEXT NOT NULL,
+        value TEXT
+    );
+    """ },
+
+    2: { "error": "old format with floating-point timestamps requires "
+         "nilmdb 1.3.1 or older" },
+
+    3: { "next": None },
+}
+
+@nilmdb.utils.must_close()
+class NilmDB(object):
+    verbose = 0
+
+    def __init__(self, basepath, max_results=None,
+                 max_removals=None, bulkdata_args=None):
+        """Initialize NilmDB at the given basepath.
+        Other arguments are for debugging / testing:
+
+        'max_results' is the max rows to send in a single
+        stream_intervals or stream_extract response.
+
+        'max_removals' is the max rows to delete at once
+        in stream_move.
+
+        'bulkdata_args' is kwargs for the bulkdata module.
+        """
+        if bulkdata_args is None:
+            bulkdata_args = {}
+
+        # set up path
+        self.basepath = os.path.abspath(basepath)
+
+        # Create the database path if it doesn't exist
+        try:
+            os.makedirs(self.basepath)
+        except OSError as e:
+            if e.errno != errno.EEXIST: # pragma: no cover
+                # (no coverage, because it's hard to trigger this case
+                # if tests are run as root)
+                raise IOError("can't create tree " + self.basepath)
+
+        # Our data goes inside it
+        self.data = bulkdata.BulkData(self.basepath, **bulkdata_args)
+
+        # SQLite database too
+        sqlfilename = os.path.join(self.basepath, "data.sql")
+        self.con = sqlite3.connect(sqlfilename, check_same_thread = True)
+        try:
+            self._sql_schema_update()
+        except Exception: # pragma: no cover
+            self.data.close()
+            raise
+
+        # See big comment at top about the performance implications of this
+        self.con.execute("PRAGMA synchronous=NORMAL")
+        self.con.execute("PRAGMA journal_mode=WAL")
+
+        # Approximate largest number of elements that we want to send
+        # in a single reply (for stream_intervals, stream_extract).
+        self.max_results = max_results or 16384
+
+        # Remove up to this many rows per call to stream_remove.
+        self.max_removals = max_removals or 1048576
+
+    def get_basepath(self):
+        return self.basepath
+
+    def close(self):
+        if self.con:
+            self.con.commit()
+            self.con.close()
+        self.data.close()
+
+    def _sql_schema_update(self):
+        cur = self.con.cursor()
+        version = cur.execute("PRAGMA user_version").fetchone()[0]
+        oldversion = version
+
+        while True:
+            if version not in _sql_schema_updates: # pragma: no cover
+                raise Exception(self.basepath + ": unknown database version "
+                                + str(version))
+            update = _sql_schema_updates[version]
+            if "error" in update: # pragma: no cover
+                raise Exception(self.basepath + ": can't use database version "
+                                + str(version) + ": " + update["error"])
+            if update["next"] is None:
+                break
+            cur.executescript(update["sql"])
+            version = update["next"]
+            if self.verbose: # pragma: no cover
+                printf("Database schema updated to %d\n", version)
+
+        if version != oldversion:
+            with self.con:
+                cur.execute("PRAGMA user_version = {v:d}".format(v=version))
+
+    def _check_user_times(self, start, end):
+        if start is None:
+            start = nilmdb.utils.time.min_timestamp
+        if end is None:
+            end = nilmdb.utils.time.max_timestamp
+        if start >= end:
+            raise NilmDBError("start must precede end")
+        return (start, end)
+
+    @nilmdb.utils.lru_cache(size = 16)
+    def _get_intervals(self, stream_id):
+        """
+        Return a mutable IntervalSet corresponding to the given stream ID.
+        """
+        iset = IntervalSet()
+        result = self.con.execute("SELECT start_time, end_time, "
+                                  "start_pos, end_pos "
+                                  "FROM ranges "
+                                  "WHERE stream_id=?", (stream_id,))
+        try:
+            for (start_time, end_time, start_pos, end_pos) in result:
+                iset += DBInterval(start_time, end_time,
+                                   start_time, end_time,
+                                   start_pos, end_pos)
+        except IntervalError: # pragma: no cover
+            raise NilmDBError("unexpected overlap in ranges table!")
+
+        return iset
+
+    def _sql_interval_insert(self, id, start, end, start_pos, end_pos):
+        """Helper that adds interval to the SQL database only"""
+        self.con.execute("INSERT INTO ranges "
+                         "(stream_id,start_time,end_time,start_pos,end_pos) "
+                         "VALUES (?,?,?,?,?)",
+                         (id, start, end, start_pos, end_pos))
+
+    def _sql_interval_delete(self, id, start, end, start_pos, end_pos):
+        """Helper that removes interval from the SQL database only"""
+        self.con.execute("DELETE FROM ranges WHERE "
+                         "stream_id=? AND start_time=? AND "
+                         "end_time=? AND start_pos=? AND end_pos=?",
+                         (id, start, end, start_pos, end_pos))
+
+    def _add_interval(self, stream_id, interval, start_pos, end_pos):
+        """
+        Add interval to the internal interval cache, and to the database.
+        Note: arguments must be ints (not numpy.int64, etc)
+        """
+        # Load this stream's intervals
+        iset = self._get_intervals(stream_id)
+
+        # Check for overlap
+        if iset.intersects(interval): # pragma: no cover (gets caught earlier)
+            raise NilmDBError("new interval overlaps existing data")
+
+        # Check for adjacency.  If there's a stream in the database
+        # that ends exactly when this one starts, and the database
+        # rows match up, we can make one interval that covers the
+        # time range [adjacent.start -> interval.end)
+        # and database rows [ adjacent.start_pos -> end_pos ].
+        # Only do this if the resulting interval isn't too large.
+        max_merged_rows = 8000 * 60 * 60 * 1.05 # 1.05 hours at 8 KHz
+        adjacent = iset.find_end(interval.start)
+        if (adjacent is not None and
+            start_pos == adjacent.db_endpos and
+            (end_pos - adjacent.db_startpos) < max_merged_rows):
+            # First delete the old one, both from our iset and the
+            # database
+            iset -= adjacent
+            self._sql_interval_delete(stream_id,
+                                      adjacent.db_start, adjacent.db_end,
+                                      adjacent.db_startpos, adjacent.db_endpos)
+
+            # Now update our interval so the fallthrough add is
+            # correct.
+            interval.start = adjacent.start
+            start_pos = adjacent.db_startpos
+
+        # Add the new interval to the iset
+        iset.iadd_nocheck(DBInterval(interval.start, interval.end,
+                                     interval.start, interval.end,
+                                     start_pos, end_pos))
+
+        # Insert into the database
+        self._sql_interval_insert(stream_id, interval.start, interval.end,
+                                  int(start_pos), int(end_pos))
+
+        self.con.commit()
+
+    def _remove_interval(self, stream_id, original, remove):
+        """
+        Remove an interval from the internal cache and the database.
+
+        stream_id: id of stream
+         original: original DBInterval; must be already present in DB
+        to_remove: DBInterval to remove; must be subset of 'original'
+        """
+        # Just return if we have nothing to remove
+        if remove.start == remove.end: # pragma: no cover
+            return
+
+        # Load this stream's intervals
+        iset = self._get_intervals(stream_id)
+
+        # Remove existing interval from the cached set and the database
+        iset -= original
+        self._sql_interval_delete(stream_id,
+                                  original.db_start, original.db_end,
+                                  original.db_startpos, original.db_endpos)
+
+        # Add back the intervals that would be left over if the
+        # requested interval is removed.  There may be two of them, if
+        # the removed piece was in the middle.
+        def add(iset, start, end, start_pos, end_pos):
+            iset += DBInterval(start, end, start, end, start_pos, end_pos)
+            self._sql_interval_insert(stream_id, start, end, start_pos, end_pos)
+
+        if original.start != remove.start:
+            # Interval before the removed region
+            add(iset, original.start, remove.start,
+                original.db_startpos, remove.db_startpos)
+
+        if original.end != remove.end:
+            # Interval after the removed region
+            add(iset, remove.end, original.end,
+                remove.db_endpos, original.db_endpos)
+
+        # Commit SQL changes
+        self.con.commit()
+
+        return
+
+    def stream_list(self, path = None, layout = None, extended = False):
+        """Return list of lists of all streams in the database.
+
+        If path is specified, include only streams with a path that
+        matches the given string.
+
+        If layout is specified, include only streams with a layout
+        that matches the given string.
+
+        If extended = False, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extended = True, returns a list of lists containing
+        more information:
+           path
+           layout
+           interval_min (earliest interval start)
+           interval_max (latest interval end)
+           rows         (total number of rows of data)
+           time         (total time covered by this stream, in timestamp units)
+        """
+        params = ()
+        query = "SELECT streams.path, streams.layout"
+        if extended:
+            query += ", min(ranges.start_time), max(ranges.end_time) "
+            query += ", coalesce(sum(ranges.end_pos - ranges.start_pos), 0) "
+            query += ", coalesce(sum(ranges.end_time - ranges.start_time), 0) "
+        query += " FROM streams"
+        if extended:
+            query += " LEFT JOIN ranges ON streams.id = ranges.stream_id"
+        query += " WHERE 1=1"
+        if layout is not None:
+            query += " AND streams.layout=?"
+            params += (layout,)
+        if path is not None:
+            query += " AND streams.path=?"
+            params += (path,)
+        query += " GROUP BY streams.id ORDER BY streams.path"
+        result = self.con.execute(query, params).fetchall()
+        return [ list(x) for x in result ]
+
+    def stream_intervals(self, path, start = None, end = None, diffpath = None):
+        """
+        List all intervals in 'path' between 'start' and 'end'.  If
+        'diffpath' is not none, list instead the set-difference
+        between the intervals in the two streams; i.e. all interval
+        ranges that are present in 'path' but not 'diffpath'.
+
+        Returns (intervals, restart) tuple.
+
+        'intervals' is a list of [start,end] timestamps of all intervals
+        that exist for path, between start and end.
+
+        'restart', if not None, means that there were too many results
+        to return in a single request.  The data is complete from the
+        starting timestamp to the point at which it was truncated, and
+        a new request with a start time of 'restart' will fetch the
+        next block of data.
+        """
+        stream_id = self._stream_id(path)
+        intervals = self._get_intervals(stream_id)
+        if diffpath:
+            diffstream_id = self._stream_id(diffpath)
+            diffintervals = self._get_intervals(diffstream_id)
+        (start, end) = self._check_user_times(start, end)
+        requested = Interval(start, end)
+        result = []
+        if diffpath:
+            getter = nilmdb.utils.interval.set_difference(
+                intervals.intersection(requested),
+                diffintervals.intersection(requested))
+        else:
+            getter = intervals.intersection(requested)
+        for n, i in enumerate(getter):
+            if n >= self.max_results:
+                restart = i.start
+                break
+            result.append([i.start, i.end])
+        else:
+            restart = None
+        return (result, restart)
+
+    def stream_create(self, path, layout_name):
+        """Create a new table in the database.
+
+        path: path to the data (e.g. '/newton/prep').
+        Paths must contain at least two elements, e.g.:
+           /newton/prep
+           /newton/raw
+           /newton/upstairs/prep
+           /newton/upstairs/raw
+
+        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
+        """
+        # Create the bulk storage.  Raises ValueError on error, which we
+        # pass along.
+        self.data.create(path, layout_name)
+
+        # Insert into SQL database once the bulk storage is happy
+        with self.con as con:
+            con.execute("INSERT INTO streams (path, layout) VALUES (?,?)",
+                        (path, layout_name))
+
+    def _stream_id(self, path):
+        """Return unique stream ID"""
+        result = self.con.execute("SELECT id FROM streams WHERE path=?",
+                                  (path,)).fetchone()
+        if result is None:
+            raise StreamError("No stream at path " + path)
+        return result[0]
+
+    def stream_set_metadata(self, path, data):
+        """Set stream metadata from a dictionary, e.g.
+           { description = 'Downstairs lighting',
+             v_scaling = 123.45 }
+           This replaces all existing metadata.
+           """
+        stream_id = self._stream_id(path)
+        with self.con as con:
+            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
+            for key in data:
+                if data[key] != '':
+                    con.execute("INSERT INTO metadata VALUES (?, ?, ?)",
+                                (stream_id, key, data[key]))
+
+    def stream_get_metadata(self, path):
+        """Return stream metadata as a dictionary."""
+        stream_id = self._stream_id(path)
+        result = self.con.execute("SELECT metadata.key, metadata.value "
+                                  "FROM metadata "
+                                  "WHERE metadata.stream_id=?", (stream_id,))
+        data = {}
+        for (key, value) in result:
+            data[key] = value
+        return data
+
+    def stream_update_metadata(self, path, newdata):
+        """Update stream metadata from a dictionary"""
+        data = self.stream_get_metadata(path)
+        data.update(newdata)
+        self.stream_set_metadata(path, data)
+
+    def stream_rename(self, oldpath, newpath):
+        """Rename a stream."""
+        stream_id = self._stream_id(oldpath)
+
+        # Rename the data
+        self.data.rename(oldpath, newpath)
+
+        # Rename the stream in the database
+        with self.con as con:
+            con.execute("UPDATE streams SET path=? WHERE id=?",
+                        (newpath, stream_id))
+
+    def stream_destroy(self, path):
+        """Fully remove a table from the database.  Fails if there are
+        any intervals data present; remove them first.  Metadata is
+        also removed."""
+        stream_id = self._stream_id(path)
+
+        # Verify that no intervals are present, and clear the cache
+        iset = self._get_intervals(stream_id)
+        if len(iset):
+            raise NilmDBError("all intervals must be removed before "
+                              "destroying a stream")
+        self._get_intervals.cache_remove(self, stream_id)
+
+        # Delete the bulkdata storage
+        self.data.destroy(path)
+
+        # Delete metadata, stream, intervals (should be none)
+        with self.con as con:
+            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
+            con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
+            con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
+
+    def stream_insert(self, path, start, end, data):
+        """Insert new data into the database.
+           path: Path at which to add the data
+           start: Starting timestamp
+           end: Ending timestamp
+           data: Textual data, formatted according to the layout of path
+           """
+        # First check for basic overlap using timestamp info given.
+        stream_id = self._stream_id(path)
+        iset = self._get_intervals(stream_id)
+        interval = Interval(start, end)
+        if iset.intersects(interval):
+            raise OverlapError("new data overlaps existing data at range: "
+                               + str(iset & interval))
+
+        # Tenatively append the data.  This will raise a ValueError if
+        # there are any parse errors.
+        table = self.data.getnode(path)
+        row_start = table.nrows
+        table.append_string(data, start, end)
+        row_end = table.nrows
+
+        # Insert the record into the sql database.
+        self._add_interval(stream_id, interval, row_start, row_end)
+
+        # And that's all
+        return
+
+    def _find_start(self, table, dbinterval):
+        """
+        Given a DBInterval, find the row in the database that
+        corresponds to the start time.  Return the first database
+        position with a timestamp (first element) greater than or
+        equal to 'start'.
+        """
+        # Optimization for the common case where an interval wasn't truncated
+        if dbinterval.start == dbinterval.db_start:
+            return dbinterval.db_startpos
+        return bisect.bisect_left(table,
+                                  dbinterval.start,
+                                  dbinterval.db_startpos,
+                                  dbinterval.db_endpos)
+
+    def _find_end(self, table, dbinterval):
+        """
+        Given a DBInterval, find the row in the database that follows
+        the end time.  Return the first database position after the
+        row with timestamp (first element) greater than or equal
+        to 'end'.
+        """
+        # Optimization for the common case where an interval wasn't truncated
+        if dbinterval.end == dbinterval.db_end:
+            return dbinterval.db_endpos
+        # Note that we still use bisect_left here, because we don't
+        # want to include the given timestamp in the results.  This is
+        # so a queries like 1:00 -> 2:00 and 2:00 -> 3:00 return
+        # non-overlapping data.
+        return bisect.bisect_left(table,
+                                  dbinterval.end,
+                                  dbinterval.db_startpos,
+                                  dbinterval.db_endpos)
+
+    def stream_extract(self, path, start = None, end = None,
+                       count = False, markup = False):
+        """
+        Returns (data, restart) tuple.
+
+        'data' is ASCII-formatted data from the database, formatted
+        according to the layout of the stream.
+
+        'restart', if not None, means that there were too many results to
+        return in a single request.  The data is complete from the
+        starting timestamp to the point at which it was truncated,
+        and a new request with a start time of 'restart' will fetch
+        the next block of data.
+
+        'count', if true, means to not return raw data, but just the count
+        of rows that would have been returned.  This is much faster
+        than actually fetching the data.  It is not limited by
+        max_results.
+
+        'markup', if true, indicates that returned data should be
+        marked with a comment denoting when a particular interval
+        starts, and another comment when an interval ends.
+        """
+        stream_id = self._stream_id(path)
+        table = self.data.getnode(path)
+        intervals = self._get_intervals(stream_id)
+        (start, end) = self._check_user_times(start, end)
+        requested = Interval(start, end)
+        result = []
+        matched = 0
+        remaining = self.max_results
+        restart = None
+        for interval in intervals.intersection(requested):
+            # Reading single rows from the table is too slow, so
+            # we use two bisections to find both the starting and
+            # ending row for this particular interval, then
+            # read the entire range as one slice.
+            row_start = self._find_start(table, interval)
+            row_end = self._find_end(table, interval)
+
+            if count:
+                matched += row_end - row_start
+                continue
+
+            # Shorten it if we'll hit the maximum number of results
+            row_max = row_start + remaining
+            if row_max < row_end:
+                row_end = row_max
+                restart = table[row_max]
+
+            # Add markup
+            if markup:
+                result.append("# interval-start " +
+                              timestamp_to_string(interval.start) + "\n")
+
+            # Gather these results up
+            result.append(table.get_data(row_start, row_end))
+
+            # Count them
+            remaining -= row_end - row_start
+
+            # Add markup, and exit if restart is set.
+            if restart is not None:
+                if markup:
+                    result.append("# interval-end " +
+                                  timestamp_to_string(restart) + "\n")
+                break
+            if markup:
+                result.append("# interval-end " +
+                              timestamp_to_string(interval.end) + "\n")
+
+        if count:
+            return matched
+        return ("".join(result), restart)
+
+    def stream_remove(self, path, start = None, end = None):
+        """
+        Remove data from the specified time interval within a stream.
+
+        Removes data in the interval [start, end), and intervals are
+        truncated or split appropriately.
+
+        Returns a (removed, restart) tuple.
+
+        'removed' is the number of data points that were removed.
+
+        'restart', if not None, means there were too many rows to
+        remove in a single request.  This function should be called
+        again with a start time of 'restart' to complete the removal.
+        """
+        stream_id = self._stream_id(path)
+        table = self.data.getnode(path)
+        intervals = self._get_intervals(stream_id)
+        (start, end) = self._check_user_times(start, end)
+        to_remove = Interval(start, end)
+        removed = 0
+        remaining = self.max_removals
+        restart = None
+
+        # Can't remove intervals from within the iterator, so we need to
+        # remember what's currently in the intersection now.
+        all_candidates = list(intervals.intersection(to_remove, orig = True))
+
+        for (dbint, orig) in all_candidates:
+            # Find row start and end
+            row_start = self._find_start(table, dbint)
+            row_end = self._find_end(table, dbint)
+
+            # Shorten it if we'll hit the maximum number of removals
+            row_max = row_start + remaining
+            if row_max < row_end:
+                row_end = row_max
+                dbint.end = table[row_max]
+                restart = dbint.end
+
+            # Adjust the DBInterval to match the newly found ends
+            dbint.db_start = dbint.start
+            dbint.db_end = dbint.end
+            dbint.db_startpos = row_start
+            dbint.db_endpos = row_end
+
+            # Remove interval from the database
+            self._remove_interval(stream_id, orig, dbint)
+
+            # Remove data from the underlying table storage
+            table.remove(row_start, row_end)
+
+            # Count how many were removed
+            removed += row_end - row_start
+
+            if restart is not None:
+                break
+
+        return (removed, restart)

nilmdb/server/rocket.c

@@ -0,0 +1,657 @@
+#include <Python.h>
+#include <structmember.h>
+#include <endian.h>
+
+#include <ctype.h>
+#include <stdint.h>
+
+/* Values missing from stdint.h */
+#define UINT8_MIN 0
+#define UINT16_MIN 0
+#define UINT32_MIN 0
+#define UINT64_MIN 0
+
+/* Marker values (if min == max, skip range check) */
+#define FLOAT32_MIN 0
+#define FLOAT32_MAX 0
+#define FLOAT64_MIN 0
+#define FLOAT64_MAX 0
+
+typedef int64_t timestamp_t;
+
+/* This code probably needs to be double-checked for the case where
+   sizeof(long) != 8, so enforce that here with something that will
+   fail at build time.  We assume that the python integer type can
+   hold an int64_t. */
+const static char __long_ok[1 - 2*!(sizeof(int64_t) ==
+				    sizeof(long int))] = { 0 };
+
+/* Somewhat arbitrary, just so we can use fixed sizes for strings
+   etc. */
+static const int MAX_LAYOUT_COUNT = 128;
+
+/* Error object and constants */
+static PyObject *ParseError;
+typedef enum {
+	ERR_OTHER,
+	ERR_NON_MONOTONIC,
+	ERR_OUT_OF_INTERVAL,
+} parseerror_code_t;
+static void add_parseerror_codes(PyObject *module)
+{
+	PyModule_AddIntMacro(module, ERR_OTHER);
+	PyModule_AddIntMacro(module, ERR_NON_MONOTONIC);
+	PyModule_AddIntMacro(module, ERR_OUT_OF_INTERVAL);
+}
+
+/* Helpers to raise ParseErrors.  Use "return raise_str(...)" etc. */
+static PyObject *raise_str(int line, int col, int code, const char *string)
+{
+	PyObject *o;
+	o = Py_BuildValue("(iiis)", line, col, code, string);
+	if (o != NULL) {
+		PyErr_SetObject(ParseError, o);
+		Py_DECREF(o);
+	}
+	return NULL;
+}
+static PyObject *raise_int(int line, int col, int code, int64_t num)
+{
+	PyObject *o;
+	o = Py_BuildValue("(iiil)", line, col, code, num);
+	if (o != NULL) {
+		PyErr_SetObject(ParseError, o);
+		Py_DECREF(o);
+	}
+	return NULL;
+}
+
+/****
+ * Layout and type helpers
+ */
+typedef union {
+	int8_t i;
+	uint8_t u;
+} union8_t;
+typedef union {
+	int16_t i;
+	uint16_t u;
+} union16_t;
+typedef union {
+	int32_t i;
+	uint32_t u;
+	float f;
+} union32_t;
+typedef union {
+	int64_t i;
+	uint64_t u;
+	double d;
+} union64_t;
+
+typedef enum {
+	LAYOUT_TYPE_NONE,
+	LAYOUT_TYPE_INT8,
+	LAYOUT_TYPE_UINT8,
+	LAYOUT_TYPE_INT16,
+	LAYOUT_TYPE_UINT16,
+	LAYOUT_TYPE_INT32,
+	LAYOUT_TYPE_UINT32,
+	LAYOUT_TYPE_INT64,
+	LAYOUT_TYPE_UINT64,
+	LAYOUT_TYPE_FLOAT32,
+	LAYOUT_TYPE_FLOAT64,
+} layout_type_t;
+
+struct {
+	char *string;
+	layout_type_t layout;
+	int size;
+} type_lookup[] = {
+	{ "int8",    LAYOUT_TYPE_INT8,    1 },
+	{ "uint8",   LAYOUT_TYPE_UINT8,   1 },
+	{ "int16",   LAYOUT_TYPE_INT16,   2 },
+	{ "uint16",  LAYOUT_TYPE_UINT16,  2 },
+	{ "int32",   LAYOUT_TYPE_INT32,   4 },
+	{ "uint32",  LAYOUT_TYPE_UINT32,  4 },
+	{ "int64",   LAYOUT_TYPE_INT64,   8 },
+	{ "uint64",  LAYOUT_TYPE_UINT64,  8 },
+	{ "float32", LAYOUT_TYPE_FLOAT32, 4 },
+	{ "float64", LAYOUT_TYPE_FLOAT64, 8 },
+	{ NULL }
+};
+
+/****
+ * Object definition, init, etc
+ */
+
+/* Rocket object */
+typedef struct {
+	PyObject_HEAD
+	layout_type_t layout_type;
+	int layout_count;
+	int binary_size;
+	FILE *file;
+	int file_size;
+} Rocket;
+
+/* Dealloc / new */
+static void Rocket_dealloc(Rocket *self)
+{
+	if (self->file) {
+		fprintf(stderr, "rocket: file wasn't closed\n");
+		fclose(self->file);
+		self->file = NULL;
+	}
+	self->ob_type->tp_free((PyObject *)self);
+}
+
+static PyObject *Rocket_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+	Rocket *self;
+
+	self = (Rocket *)type->tp_alloc(type, 0);
+	if (!self)
+		return NULL;
+	self->layout_type = LAYOUT_TYPE_NONE;
+	self->layout_count = 0;
+	self->binary_size = 0;
+	self->file = NULL;
+	self->file_size = -1;
+	return (PyObject *)self;
+}
+
+/* .__init__(layout, file) */
+static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
+{
+	const char *layout, *path;
+	static char *kwlist[] = { "layout", "file", NULL };
+	if (!PyArg_ParseTupleAndKeywords(args, kwds, "sz", kwlist,
+					 &layout, &path))
+		return -1;
+	if (!layout)
+		return -1;
+	if (path) {
+		if ((self->file = fopen(path, "a+b")) == NULL) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return -1;
+		}
+		self->file_size = -1;
+	} else {
+		self->file = NULL;
+	}
+
+	const char *under;
+	char *tmp;
+	under = strchr(layout, '_');
+	if (!under) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"badly formatted string");
+		return -1;
+	}
+	self->layout_count = strtoul(under+1, &tmp, 10);
+	if (self->layout_count < 1 || *tmp != '\0') {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"bad count");
+		return -1;
+	}
+	if (self->layout_count >= MAX_LAYOUT_COUNT) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"count too high");
+		return -1;
+	}
+
+	int i;
+	for (i = 0; type_lookup[i].string; i++)
+		if (strncmp(layout, type_lookup[i].string, under-layout) == 0)
+			break;
+	if (!type_lookup[i].string) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"bad data type");
+		return -1;
+	}
+	self->layout_type = type_lookup[i].layout;
+	self->binary_size = 8 + (type_lookup[i].size * self->layout_count);
+
+	return 0;
+}
+
+/* .close() */
+static PyObject *Rocket_close(Rocket *self)
+{
+	if (self->file) {
+		fclose(self->file);
+		self->file = NULL;
+	}
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+/* .file_size property */
+static PyObject *Rocket_get_file_size(Rocket *self)
+{
+	if (!self->file) {
+		PyErr_SetString(PyExc_AttributeError, "no file");
+		return NULL;
+	}
+	if (self->file_size < 0) {
+		int oldpos;
+		if (((oldpos = ftell(self->file)) < 0) ||
+		    (fseek(self->file, 0, SEEK_END) < 0) ||
+		    ((self->file_size = ftell(self->file)) < 0) ||
+		    (fseek(self->file, oldpos, SEEK_SET) < 0)) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return NULL;
+		}
+	}
+	return PyInt_FromLong(self->file_size);
+}
+
+/****
+ * Append from string
+ */
+static inline long int strtol10(const char *nptr, char **endptr) {
+	return strtol(nptr, endptr, 10);
+}
+static inline long int strtoul10(const char *nptr, char **endptr) {
+	return strtoul(nptr, endptr, 10);
+}
+
+/* .append_string(count, data, offset, linenum, start, end, last_timestamp) */
+static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
+{
+	int count;
+	const char *data;
+	int offset;
+	const char *linestart;
+	int linenum;
+	timestamp_t start;
+	timestamp_t end;
+	timestamp_t last_timestamp;
+
+	int written = 0;
+	char *endptr;
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+	int i;
+
+	/* It would be nice to use 't#' instead of 's' for data,
+	   but we need the null termination for strto*.  If we had
+	   strnto* that took a length, we could use t# and not require
+	   a copy. */
+	if (!PyArg_ParseTuple(args, "isiilll:append_string", &count,
+			      &data, &offset, &linenum,
+			      &start, &end, &last_timestamp))
+		return NULL;
+
+	/* Skip spaces, but don't skip over a newline. */
+#define SKIP_BLANK(buf) do {			\
+	while (isspace(*buf)) {			\
+		if (*buf == '\n')		\
+			break;			\
+		buf++;				\
+	} } while(0)
+
+	const char *buf = &data[offset];
+	while (written < count && *buf)
+	{
+		linestart = buf;
+		linenum++;
+
+		/* Skip leading whitespace and commented lines */
+		SKIP_BLANK(buf);
+		if (*buf == '#') {
+			while (*buf && *buf != '\n')
+				buf++;
+			if (*buf)
+				buf++;
+			continue;
+		}
+
+		/* Extract timestamp */
+		t64.i = strtoll(buf, &endptr, 10);
+		if (endptr == buf || !isspace(*endptr)) {
+			/* Try parsing as a double instead */
+			t64.d = strtod(buf, &endptr);
+			if (endptr == buf)
+				goto bad_timestamp;
+			if (!isspace(*endptr))
+				goto cant_parse_value;
+			t64.i = round(t64.d);
+		}
+		if (t64.i <= last_timestamp)
+			return raise_int(linenum, buf - linestart + 1,
+					 ERR_NON_MONOTONIC, t64.i);
+		last_timestamp = t64.i;
+		if (t64.i < start || t64.i >= end)
+			return raise_int(linenum, buf - linestart + 1,
+					 ERR_OUT_OF_INTERVAL, t64.i);
+		t64.u = le64toh(t64.u);
+		if (fwrite(&t64.u, 8, 1, self->file) != 1)
+			goto err;
+		buf = endptr;
+
+		/* Parse all values in the line */
+		switch (self->layout_type) {
+#define CS(type, parsefunc, parsetype, realtype, disktype, letoh, bytes) \
+		case LAYOUT_TYPE_##type:				\
+			/* parse and write in a loop */			\
+			for (i = 0; i < self->layout_count; i++) {	\
+				/* skip non-newlines */			\
+				SKIP_BLANK(buf);			\
+				if (*buf == '\n')			\
+					goto wrong_number_of_values;	\
+				/* parse number */			\
+				parsetype = parsefunc(buf, &endptr);	\
+				if (*endptr && !isspace(*endptr))	\
+					goto cant_parse_value;		\
+				/* check limits */			\
+				if (type##_MIN != type##_MAX &&		\
+				    (parsetype < type##_MIN ||		\
+				     parsetype > type##_MAX))		\
+					goto value_out_of_range;	\
+				/* convert to disk representation */	\
+				realtype = parsetype;			\
+				disktype = letoh(disktype);		\
+				/* write it */				\
+				if (fwrite(&disktype, bytes,		\
+					   1, self->file) != 1)		\
+					goto err;			\
+				/* advance buf */			\
+				buf = endptr;				\
+			}						\
+			/* Skip trailing whitespace and comments */	\
+			SKIP_BLANK(buf);				\
+			if (*buf == '#')				\
+				while (*buf && *buf != '\n')		\
+					buf++;				\
+			if (*buf == '\n')				\
+				buf++;					\
+			else if (*buf != '\0')				\
+				goto extra_data_on_line;		\
+			break
+
+			CS(INT8,   strtol10,  t64.i, t8.i,  t8.u,         , 1);
+			CS(UINT8,  strtoul10, t64.u, t8.u,  t8.u,         , 1);
+			CS(INT16,  strtol10,  t64.i, t16.i, t16.u, le16toh, 2);
+			CS(UINT16, strtoul10, t64.u, t16.u, t16.u, le16toh, 2);
+			CS(INT32,  strtol10,  t64.i, t32.i, t32.u, le32toh, 4);
+			CS(UINT32, strtoul10, t64.u, t32.u, t32.u, le32toh, 4);
+			CS(INT64,  strtol10,  t64.i, t64.i, t64.u, le64toh, 8);
+			CS(UINT64, strtoul10, t64.u, t64.u, t64.u, le64toh, 8);
+			CS(FLOAT32, strtod,   t64.d, t32.f, t32.u, le32toh, 4);
+			CS(FLOAT64, strtod,   t64.d, t64.d, t64.u, le64toh, 8);
+#undef CS
+		default:
+			PyErr_SetString(PyExc_TypeError, "unknown type");
+			return NULL;
+		}
+
+		/* Done this line */
+		written++;
+	}
+
+	fflush(self->file);
+
+	/* Build return value and return */
+	offset = buf - data;
+	PyObject *o;
+	o = Py_BuildValue("(iili)", written, offset, last_timestamp, linenum);
+	return o;
+err:
+	PyErr_SetFromErrno(PyExc_OSError);
+	return NULL;
+bad_timestamp:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "bad timestamp");
+cant_parse_value:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "can't parse value");
+wrong_number_of_values:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "wrong number of values");
+value_out_of_range:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "value out of range");
+extra_data_on_line:
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "extra data on line");
+}
+
+/****
+ * Extract to string
+ */
+
+static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
+{
+	long count;
+	long offset;
+
+	if (!PyArg_ParseTuple(args, "ll", &offset, &count))
+		return NULL;
+	if (!self->file) {
+		PyErr_SetString(PyExc_Exception, "no file");
+		return NULL;
+	}
+	/* Seek to target location */
+	if (fseek(self->file, offset, SEEK_SET) < 0) {
+		PyErr_SetFromErrno(PyExc_OSError);
+		return NULL;
+	}
+
+	char *str = NULL, *new;
+	long len_alloc = 0;
+	long len = 0;
+	int ret;
+
+	/* min space free in string (and the maximum length of one
+	   line); this is generous */
+	const int min_free = 32 * MAX_LAYOUT_COUNT;
+
+	/* how much to allocate at once */
+	const int alloc_size = 1048576;
+
+	int row, i;
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+	for (row = 0; row < count; row++) {
+		/* Make sure there's space for a line */
+		if ((len_alloc - len) < min_free) {
+			/* grow by 1 meg at a time */
+			len_alloc += alloc_size;
+			new = realloc(str, len_alloc);
+			if (new == NULL)
+				goto err;
+			str = new;
+		}
+
+		/* Read and print timestamp */
+		if (fread(&t64.u, 8, 1, self->file) != 1)
+			goto err;
+		t64.u = le64toh(t64.u);
+		ret = sprintf(&str[len], "%ld", t64.i);
+		if (ret <= 0)
+			goto err;
+		len += ret;
+
+		/* Read and print values */
+		switch (self->layout_type) {
+#define CASE(type, fmt, fmttype, disktype, letoh, bytes)		\
+		case LAYOUT_TYPE_##type:				\
+			/* read and format in a loop */			\
+			for (i = 0; i < self->layout_count; i++) {	\
+				if (fread(&disktype, bytes,		\
+					  1, self->file) < 0)		\
+					goto err;			\
+				disktype = letoh(disktype);		\
+				ret = sprintf(&str[len], " " fmt,	\
+					      fmttype);			\
+				if (ret <= 0)				\
+					goto err;			\
+				len += ret;				\
+			}						\
+			break
+			CASE(INT8,   "%hhd",   t8.i,  t8.u,         , 1);
+			CASE(UINT8,  "%hhu",   t8.u,  t8.u,         , 1);
+			CASE(INT16,  "%hd",    t16.i, t16.u, le16toh, 2);
+			CASE(UINT16, "%hu",    t16.u, t16.u, le16toh, 2);
+			CASE(INT32,  "%d",     t32.i, t32.u, le32toh, 4);
+			CASE(UINT32, "%u",     t32.u, t32.u, le32toh, 4);
+			CASE(INT64,  "%ld",    t64.i, t64.u, le64toh, 8);
+			CASE(UINT64, "%lu",    t64.u, t64.u, le64toh, 8);
+			/* These next two are a bit debatable.  floats
+			   are 6-9 significant figures, so we print 7.
+			   Doubles are 15-19, so we print 17.  This is
+			   similar to the old prep format for float32.
+			*/
+			CASE(FLOAT32, "%.6e",  t32.f, t32.u, le32toh, 4);
+			CASE(FLOAT64, "%.16e", t64.d, t64.u, le64toh, 8);
+#undef CASE
+		default:
+			PyErr_SetString(PyExc_TypeError, "unknown type");
+			if (str) free(str);
+			return NULL;
+		}
+		str[len++] = '\n';
+	}
+
+	PyObject *pystr = PyString_FromStringAndSize(str, len);
+	free(str);
+	return pystr;
+err:
+	if (str) free(str);
+	PyErr_SetFromErrno(PyExc_OSError);
+	return NULL;
+}
+
+
+/****
+ * Extract timestamp
+ */
+static PyObject *Rocket_extract_timestamp(Rocket *self, PyObject *args)
+{
+	long offset;
+	union64_t t64;
+	if (!PyArg_ParseTuple(args, "l", &offset))
+		return NULL;
+	if (!self->file) {
+		PyErr_SetString(PyExc_Exception, "no file");
+		return NULL;
+	}
+
+	/* Seek to target location and read timestamp */
+	if ((fseek(self->file, offset, SEEK_SET) < 0) ||
+	    (fread(&t64.u, 8, 1, self->file) != 1)) {
+		PyErr_SetFromErrno(PyExc_OSError);
+		return NULL;
+	}
+
+	/* Convert and return */
+	t64.u = le64toh(t64.u);
+	return Py_BuildValue("l", t64.i);
+}
+
+/****
+ * Module and type setup
+ */
+
+static PyGetSetDef Rocket_getsetters[] = {
+	{ "file_size", (getter)Rocket_get_file_size, NULL,
+	  "file size in bytes", NULL },
+	{ NULL },
+};
+
+static PyMemberDef Rocket_members[] = {
+	{ "binary_size", T_INT, offsetof(Rocket, binary_size), 0,
+	  "binary size per row" },
+	{ NULL },
+};
+
+static PyMethodDef Rocket_methods[] = {
+	{ "close", (PyCFunction)Rocket_close, METH_NOARGS,
+	  "close(self)\n\n"
+	  "Close file handle" },
+
+	{ "append_string", (PyCFunction)Rocket_append_string, METH_VARARGS,
+	  "append_string(self, count, data, offset, line, start, end, ts)\n\n"
+          "Parse string and append data.\n"
+	  "\n"
+	  "  count: maximum number of rows to add\n"
+          "  data: string data\n"
+          "  offset: byte offset into data to start parsing\n"
+          "  line: current line number of data\n"
+          "  start: starting timestamp for interval\n"
+          "  end: end timestamp for interval\n"
+          "  ts: last timestamp that was previously parsed\n"
+	  "\n"
+	  "Raises ParseError if timestamps are non-monotonic, outside\n"
+	  "the start/end interval etc.\n"
+	  "\n"
+          "On success, return a tuple with three values:\n"
+          "  added_rows: how many rows were added from the file\n"
+          "  data_offset: current offset into the data string\n"
+          "  last_timestamp: last timestamp we parsed" },
+
+	{ "extract_string", (PyCFunction)Rocket_extract_string, METH_VARARGS,
+	  "extract_string(self, offset, count)\n\n"
+	  "Extract count rows of data from the file at offset offset.\n"
+	  "Return an ascii formatted string according to the layout" },
+
+	{ "extract_timestamp",
+	  (PyCFunction)Rocket_extract_timestamp, METH_VARARGS,
+	  "extract_timestamp(self, offset)\n\n"
+	  "Extract a single timestamp from the file" },
+
+	{ NULL },
+};
+
+static PyTypeObject RocketType = {
+	PyObject_HEAD_INIT(NULL)
+
+	.tp_name	= "rocket.Rocket",
+	.tp_basicsize	= sizeof(Rocket),
+	.tp_flags	= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+
+	.tp_new		= Rocket_new,
+	.tp_dealloc	= (destructor)Rocket_dealloc,
+	.tp_init	= (initproc)Rocket_init,
+	.tp_methods	= Rocket_methods,
+	.tp_members	= Rocket_members,
+	.tp_getset	= Rocket_getsetters,
+
+	.tp_doc		= ("rocket.Rocket(layout, file)\n\n"
+			   "C implementation of the \"rocket\" data parsing\n"
+			   "interface, which translates between the binary\n"
+			   "format on disk and the ASCII or Python list\n"
+			   "format used when communicating with the rest of\n"
+			   "the system.")
+};
+
+static PyMethodDef module_methods[] = {
+	{ NULL },
+};
+
+PyMODINIT_FUNC
+initrocket(void)
+{
+	PyObject *module;
+
+	RocketType.tp_new = PyType_GenericNew;
+	if (PyType_Ready(&RocketType) < 0)
+		return;
+
+	module = Py_InitModule3("rocket", module_methods,
+				"Rocket data parsing and formatting module");
+	Py_INCREF(&RocketType);
+	PyModule_AddObject(module, "Rocket", (PyObject *)&RocketType);
+
+	ParseError = PyErr_NewException("rocket.ParseError", NULL, NULL);
+	Py_INCREF(ParseError);
+	PyModule_AddObject(module, "ParseError", ParseError);
+	add_parseerror_codes(module);
+
+	return;
+}

nilmdb/server/server.py

@@ -0,0 +1,649 @@
+"""CherryPy-based server for accessing NILM database via HTTP"""
+
+# Need absolute_import so that "import nilmdb" won't pull in
+# nilmdb.py, but will pull the nilmdb module instead.
+from __future__ import absolute_import
+import nilmdb.server
+from nilmdb.utils.printf import *
+from nilmdb.server.errors import NilmDBError
+from nilmdb.utils.time import string_to_timestamp
+
+import cherrypy
+import sys
+import os
+import socket
+import simplejson as json
+import decorator
+import psutil
+import traceback
+
+class NilmApp(object):
+    def __init__(self, db):
+        self.db = db
+
+# Decorators
+def chunked_response(func):
+    """Decorator to enable chunked responses."""
+    # Set this to False to get better tracebacks from some requests
+    # (/stream/extract, /stream/intervals).
+    func._cp_config = { 'response.stream': True }
+    return func
+
+def response_type(content_type):
+    """Return a decorator-generating function that sets the
+    response type to the specified string."""
+    def wrapper(func, *args, **kwargs):
+        cherrypy.response.headers['Content-Type'] = content_type
+        return func(*args, **kwargs)
+    return decorator.decorator(wrapper)
+
+@decorator.decorator
+def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
+    """Decorator to work around CherryPy bug #1200 in a response
+    generator.
+
+    Even if chunked responses are disabled, LookupError or
+    UnicodeError exceptions may still be swallowed by CherryPy due to
+    bug #1200.  This throws them as generic Exceptions instead so that
+    they make it through.
+    """
+    exc_info = None
+    try:
+        for val in func(*args, **kwargs):
+            yield val
+    except (LookupError, UnicodeError):
+        # Re-raise it, but maintain the original traceback
+        exc_info = sys.exc_info()
+        new_exc = Exception(exc_info[0].__name__ + ": " + str(exc_info[1]))
+        raise new_exc, None, exc_info[2]
+    finally:
+        del exc_info
+
+def exception_to_httperror(*expected):
+    """Return a decorator-generating function that catches expected
+    errors and throws a HTTPError describing it instead.
+
+        @exception_to_httperror(NilmDBError, ValueError)
+        def foo():
+            pass
+    """
+    def wrapper(func, *args, **kwargs):
+        exc_info = None
+        try:
+            return func(*args, **kwargs)
+        except expected:
+            # Re-raise it, but maintain the original traceback
+            exc_info = sys.exc_info()
+            new_exc = cherrypy.HTTPError("400 Bad Request", str(exc_info[1]))
+            raise new_exc, None, exc_info[2]
+        finally:
+            del exc_info
+    # We need to preserve the function's argspecs for CherryPy to
+    # handle argument errors correctly.  Decorator.decorator takes
+    # care of that.
+    return decorator.decorator(wrapper)
+
+# Custom CherryPy tools
+
+def CORS_allow(methods):
+    """This does several things:
+
+    Handles CORS preflight requests.
+    Adds Allow: header to all requests.
+    Raise 405 if request.method not in method.
+
+    It is similar to cherrypy.tools.allow, with the CORS stuff added.
+    """
+    request = cherrypy.request.headers
+    response = cherrypy.response.headers
+
+    if not isinstance(methods, (tuple, list)): # pragma: no cover
+        methods = [ methods ]
+    methods = [ m.upper() for m in methods if m ]
+    if not methods: # pragma: no cover
+        methods = [ 'GET', 'HEAD' ]
+    elif 'GET' in methods and 'HEAD' not in methods: # pragma: no cover
+        methods.append('HEAD')
+    response['Allow'] = ', '.join(methods)
+
+    # Allow all origins
+    if 'Origin' in request:
+        response['Access-Control-Allow-Origin'] = request['Origin']
+
+    # If it's a CORS request, send response.
+    request_method = request.get("Access-Control-Request-Method", None)
+    request_headers = request.get("Access-Control-Request-Headers", None)
+    if (cherrypy.request.method == "OPTIONS" and
+        request_method and request_headers):
+        response['Access-Control-Allow-Headers'] = request_headers
+        response['Access-Control-Allow-Methods'] = ', '.join(methods)
+        # Try to stop further processing and return a 200 OK
+        cherrypy.response.status = "200 OK"
+        cherrypy.response.body = ""
+        cherrypy.request.handler = lambda: ""
+        return
+
+    # Reject methods that were not explicitly allowed
+    if cherrypy.request.method not in methods:
+        raise cherrypy.HTTPError(405)
+
+cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
+
+# Helper for json_in tool to process JSON data into normal request
+# parameters.
+def json_to_request_params(body):
+    cherrypy.lib.jsontools.json_processor(body)
+    if not isinstance(cherrypy.request.json, dict):
+        raise cherrypy.HTTPError(415)
+    cherrypy.request.params.update(cherrypy.request.json)
+
+# CherryPy apps
+class Root(NilmApp):
+    """Root application for NILM database"""
+
+    def __init__(self, db):
+        super(Root, self).__init__(db)
+
+    # /
+    @cherrypy.expose
+    def index(self):
+        raise cherrypy.NotFound()
+
+    # /favicon.ico
+    @cherrypy.expose
+    def favicon_ico(self):
+        raise cherrypy.NotFound()
+
+    # /version
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def version(self):
+        return nilmdb.__version__
+
+    # /dbinfo
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def dbinfo(self):
+        """Return a dictionary with the database path,
+        size of the database in bytes, and free disk space in bytes"""
+        path = self.db.get_basepath()
+        return { "path": path,
+                 "size": nilmdb.utils.du(path),
+                 "free": psutil.disk_usage(path).free }
+
+class Stream(NilmApp):
+    """Stream-specific operations"""
+
+    # Helpers
+    def _get_times(self, start_param, end_param):
+        (start, end) = (None, None)
+        if start_param is not None:
+            start = string_to_timestamp(start_param)
+        if end_param is not None:
+            end = string_to_timestamp(end_param)
+        if start is not None and end is not None:
+            if start >= end:
+                raise cherrypy.HTTPError(
+                    "400 Bad Request",
+                    sprintf("start must precede end (%s >= %s)",
+                            start_param, end_param))
+        return (start, end)
+
+    # /stream/list
+    # /stream/list?layout=float32_8
+    # /stream/list?path=/newton/prep&extended=1
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def list(self, path = None, layout = None, extended = None):
+        """List all streams in the database.  With optional path or
+        layout parameter, just list streams that match the given path
+        or layout.
+
+        If extent is not given, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extended is provided, returns a list of lists containing
+        extended info: [ path, layout, extent_min, extent_max,
+        total_rows, total_seconds ].  More data may be added.
+        """
+        return self.db.stream_list(path, layout, bool(extended))
+
+    # /stream/create?path=/newton/prep&layout=float32_8
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    def create(self, path, layout):
+        """Create a new stream in the database.  Provide path
+        and one of the nilmdb.layout.layouts keys.
+        """
+        return self.db.stream_create(path, layout)
+
+    # /stream/destroy?path=/newton/prep
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    def destroy(self, path):
+        """Delete a stream.  Fails if any data is still present."""
+        return self.db.stream_destroy(path)
+
+    # /stream/rename?oldpath=/newton/prep&newpath=/newton/prep/1
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    def rename(self, oldpath, newpath):
+        """Rename a stream."""
+        return self.db.stream_rename(oldpath, newpath)
+
+    # /stream/get_metadata?path=/newton/prep
+    # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def get_metadata(self, path, key=None):
+        """Get metadata for the named stream.  If optional
+        key parameters are specified, only return metadata
+        matching the given keys."""
+        try:
+            data = self.db.stream_get_metadata(path)
+        except nilmdb.server.nilmdb.StreamError as e:
+            raise cherrypy.HTTPError("404 Not Found", e.message)
+        if key is None:  # If no keys specified, return them all
+            key = data.keys()
+        elif not isinstance(key, list):
+            key = [ key ]
+        result = {}
+        for k in key:
+            if k in data:
+                result[k] = data[k]
+            else: # Return "None" for keys with no matching value
+                result[k] = None
+        return result
+
+    # Helper for set_metadata and get_metadata
+    def _metadata_helper(self, function, path, data):
+        if not isinstance(data, dict):
+            try:
+                data = dict(json.loads(data))
+            except TypeError as e:
+                raise NilmDBError("can't parse 'data' parameter: " + e.message)
+        for key in data:
+            if not (isinstance(data[key], basestring) or
+                    isinstance(data[key], float) or
+                    isinstance(data[key], int)):
+                raise NilmDBError("metadata values must be a string or number")
+        function(path, data)
+
+    # /stream/set_metadata?path=/newton/prep&data=<json>
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError, LookupError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    def set_metadata(self, path, data):
+        """Set metadata for the named stream, replacing any existing
+        metadata.  Data can be json-encoded or a plain dictionary."""
+        self._metadata_helper(self.db.stream_set_metadata, path, data)
+
+    # /stream/update_metadata?path=/newton/prep&data=<json>
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError, LookupError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    def update_metadata(self, path, data):
+        """Set metadata for the named stream, replacing any existing
+        metadata.  Data can be json-encoded or a plain dictionary."""
+        self._metadata_helper(self.db.stream_update_metadata, path, data)
+
+    # /stream/insert?path=/newton/prep
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["PUT"])
+    def insert(self, path, start, end):
+        """
+        Insert new data into the database.  Provide textual data
+        (matching the path's layout) as a HTTP PUT.
+        """
+        # Important that we always read the input before throwing any
+        # errors, to keep lengths happy for persistent connections.
+        # Note that CherryPy 3.2.2 has a bug where this fails for GET
+        # requests, if we ever want to handle those (issue #1134)
+        body = cherrypy.request.body.read()
+
+        # Check path and get layout
+        streams = self.db.stream_list(path = path)
+        if len(streams) != 1:
+            raise cherrypy.HTTPError("404 Not Found", "No such stream")
+
+        # Check limits
+        (start, end) = self._get_times(start, end)
+
+        # Pass the data directly to nilmdb, which will parse it and
+        # raise a ValueError if there are any problems.
+        self.db.stream_insert(path, start, end, body)
+
+        # Done
+        return
+
+    # /stream/remove?path=/newton/prep
+    # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
+    @cherrypy.expose
+    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_out()
+    @exception_to_httperror(NilmDBError)
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    def remove(self, path, start = None, end = None):
+        """
+        Remove data from the backend database.  Removes all data in
+        the interval [start, end).  Returns the number of data points
+        removed.
+        """
+        (start, end) = self._get_times(start, end)
+        total_removed = 0
+        while True:
+            (removed, restart) = self.db.stream_remove(path, start, end)
+            total_removed += removed
+            if restart is None:
+                break
+            start = restart
+        return total_removed
+
+    # /stream/intervals?path=/newton/prep
+    # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
+    # /stream/intervals?path=/newton/prep&diffpath=/newton/prep2
+    @cherrypy.expose
+    @chunked_response
+    @response_type("application/x-json-stream")
+    def intervals(self, path, start = None, end = None, diffpath = None):
+        """
+        Get intervals from backend database.  Streams the resulting
+        intervals as JSON strings separated by CR LF pairs.  This may
+        make multiple requests to the nilmdb backend to avoid causing
+        it to block for too long.
+
+        Returns intervals between 'start' and 'end' belonging to
+        'path'.  If 'diff' is provided, the set-difference between
+        intervals in 'path' and intervals in 'diffpath' are
+        returned instead.
+
+        Note that the response type is the non-standard
+        'application/x-json-stream' for lack of a better option.
+        """
+        (start, end) = self._get_times(start, end)
+
+        if len(self.db.stream_list(path = path)) != 1:
+            raise cherrypy.HTTPError("404", "No such stream: " + path)
+
+        if diffpath and len(self.db.stream_list(path = diffpath)) != 1:
+            raise cherrypy.HTTPError("404", "No such stream: " + diffpath)
+
+        @workaround_cp_bug_1200
+        def content(start, end):
+            # Note: disable chunked responses to see tracebacks from here.
+            while True:
+                (ints, restart) = self.db.stream_intervals(path, start, end,
+                                                           diffpath)
+                response = ''.join([ json.dumps(i) + "\r\n" for i in ints ])
+                yield response
+                if restart is None:
+                    break
+                start = restart
+        return content(start, end)
+
+    # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
+    @cherrypy.expose
+    @chunked_response
+    @response_type("text/plain")
+    def extract(self, path, start = None, end = None,
+                count = False, markup = False):
+        """
+        Extract data from backend database.  Streams the resulting
+        entries as ASCII text lines separated by newlines.  This may
+        make multiple requests to the nilmdb backend to avoid causing
+        it to block for too long.
+
+        If 'count' is True, returns a count rather than actual data.
+
+        If 'markup' is True, adds comments to the stream denoting each
+        interval's start and end timestamp.
+        """
+        (start, end) = self._get_times(start, end)
+
+        # Check path and get layout
+        streams = self.db.stream_list(path = path)
+        if len(streams) != 1:
+            raise cherrypy.HTTPError("404 Not Found", "No such stream")
+
+        @workaround_cp_bug_1200
+        def content(start, end):
+            # Note: disable chunked responses to see tracebacks from here.
+            if count:
+                matched = self.db.stream_extract(path, start, end,
+                                                 count = True)
+                yield sprintf("%d\n", matched)
+                return
+
+            while True:
+                (data, restart) = self.db.stream_extract(
+                    path, start, end, count = False, markup = markup)
+                yield data
+
+                if restart is None:
+                    return
+                start = restart
+        return content(start, end)
+
+class Exiter(object):
+    """App that exits the server, for testing"""
+    @cherrypy.expose
+    def index(self):
+        cherrypy.response.headers['Content-Type'] = 'text/plain'
+        def content():
+            yield 'Exiting by request'
+            raise SystemExit
+        return content()
+    index._cp_config = { 'response.stream': True }
+
+class Server(object):
+    def __init__(self, db, host = '127.0.0.1', port = 8080,
+                 stoppable = False,       # whether /exit URL exists
+                 embedded = True,         # hide diagnostics and output, etc
+                 fast_shutdown = False,   # don't wait for clients to disconn.
+                 force_traceback = False, # include traceback in all errors
+                 basepath = '',           # base URL path for cherrypy.tree
+                 ):
+        # Save server version, just for verification during tests
+        self.version = nilmdb.__version__
+
+        self.embedded = embedded
+        self.db = db
+        if not getattr(db, "_thread_safe", None):
+            raise KeyError("Database object " + str(db) + " doesn't claim "
+                           "to be thread safe.  You should pass "
+                           "nilmdb.utils.serializer_proxy(NilmDB)(args) "
+                           "rather than NilmDB(args).")
+
+        # Build up global server configuration
+        cherrypy.config.update({
+            'server.socket_host': host,
+            'server.socket_port': port,
+            'engine.autoreload_on': False,
+            'server.max_request_body_size': 8*1024*1024,
+            })
+        if self.embedded:
+            cherrypy.config.update({ 'environment': 'embedded' })
+
+        # Build up application specific configuration
+        app_config = {}
+        app_config.update({
+            'error_page.default': self.json_error_page,
+            })
+
+        # Some default headers to just help identify that things are working
+        app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
+
+        # Set up Cross-Origin Resource Sharing (CORS) handler so we
+        # can correctly respond to browsers' CORS preflight requests.
+        # This also limits verbs to GET and HEAD by default.
+        app_config.update({ 'tools.CORS_allow.on': True,
+                            'tools.CORS_allow.methods': ['GET', 'HEAD'] })
+
+        # Configure the 'json_in' tool to also allow other content-types
+        # (like x-www-form-urlencoded), and to treat JSON as a dict that
+        # fills requests.param.
+        app_config.update({ 'tools.json_in.force': False,
+                            'tools.json_in.processor': json_to_request_params })
+
+        # Send tracebacks in error responses.  They're hidden by the
+        # error_page function for client errors (code 400-499).
+        app_config.update({ 'request.show_tracebacks' : True })
+        self.force_traceback = force_traceback
+
+        # Patch CherryPy error handler to never pad out error messages.
+        # This isn't necessary, but then again, neither is padding the
+        # error messages.
+        cherrypy._cperror._ie_friendly_error_sizes = {}
+
+        # Build up the application and mount it
+        root = Root(self.db)
+        root.stream = Stream(self.db)
+        if stoppable:
+            root.exit = Exiter()
+        cherrypy.tree.apps = {}
+        cherrypy.tree.mount(root, basepath, config = { "/" : app_config })
+
+        # Shutdowns normally wait for clients to disconnect.  To speed
+        # up tests, set fast_shutdown = True
+        if fast_shutdown:
+            # Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
+            cherrypy.server.shutdown_timeout = 0.01
+        else:
+            cherrypy.server.shutdown_timeout = 5
+
+        # Set up the WSGI application pointer for external programs
+        self.wsgi_application = cherrypy.tree
+
+    def json_error_page(self, status, message, traceback, version):
+        """Return a custom error page in JSON so the client can parse it"""
+        errordata = { "status" : status,
+                      "message" : message,
+                      "traceback" : traceback }
+        # Don't send a traceback if the error was 400-499 (client's fault)
+        try:
+            code = int(status.split()[0])
+            if not self.force_traceback:
+                if code >= 400 and code <= 499:
+                    errordata["traceback"] = ""
+        except Exception: # pragma: no cover
+            pass
+        # Override the response type, which was previously set to text/html
+        cherrypy.serving.response.headers['Content-Type'] = (
+            "application/json;charset=utf-8" )
+        # Undo the HTML escaping that cherrypy's get_error_page function applies
+        # (cherrypy issue 1135)
+        for k, v in errordata.iteritems():
+            v = v.replace("&lt;","<")
+            v = v.replace("&gt;",">")
+            v = v.replace("&amp;","&")
+            errordata[k] = v
+        return json.dumps(errordata, separators=(',',':'))
+
+    def start(self, blocking = False, event = None):
+
+        if not self.embedded: # pragma: no cover
+            # Handle signals nicely
+            if hasattr(cherrypy.engine, "signal_handler"):
+                cherrypy.engine.signal_handler.subscribe()
+            if hasattr(cherrypy.engine, "console_control_handler"):
+                cherrypy.engine.console_control_handler.subscribe()
+
+        # Cherrypy stupidly calls os._exit(70) when it can't bind the
+        # port.  At least try to print a reasonable error and continue
+        # in this case, rather than just dying silently (as we would
+        # otherwise do in embedded mode)
+        real_exit = os._exit
+        def fake_exit(code): # pragma: no cover
+            if code == os.EX_SOFTWARE:
+                fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
+            else:
+                real_exit(code)
+        os._exit = fake_exit
+        cherrypy.engine.start()
+        os._exit = real_exit
+
+        # Signal that the engine has started successfully
+        if event is not None:
+            event.set()
+
+        if blocking:
+            try:
+                cherrypy.engine.wait(cherrypy.engine.states.EXITING,
+                                     interval = 0.1, channel = 'main')
+            except (KeyboardInterrupt, IOError): # pragma: no cover
+                cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
+                cherrypy.engine.exit()
+            except SystemExit: # pragma: no cover
+                cherrypy.engine.log('SystemExit raised: shutting down bus')
+                cherrypy.engine.exit()
+                raise
+
+    def stop(self):
+        cherrypy.engine.exit()
+
+# Use a single global nilmdb.server.NilmDB and nilmdb.server.Server
+# instance since the database can only be opened once.  For this to
+# work, the web server must use only a single process and single
+# Python interpreter.  Multiple threads are OK.
+_wsgi_server = None
+def wsgi_application(dbpath, basepath): # pragma: no cover
+    """Return a WSGI application object with a database at the
+    specified path.
+
+    'dbpath' is a filesystem location, e.g. /home/nilm/db
+
+    'basepath' is the URL path of the application base, which
+    is the same as the first argument to Apache's WSGIScriptAlias
+    directive.
+    """
+    def application(environ, start_response):
+        global _wsgi_server
+        if _wsgi_server is None:
+            # Try to start the server
+            try:
+                db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(dbpath)
+                _wsgi_server = nilmdb.server.Server(
+                    db, embedded = True,
+                    basepath = basepath.rstrip('/'))
+            except Exception:
+                # Build an error message on failure
+                import pprint
+                err = sprintf("Initializing database at path '%s' failed:\n\n",
+                              dbpath)
+                err += traceback.format_exc()
+                try:
+                    import pwd
+                    import grp
+                    err += sprintf("\nRunning as: uid=%d (%s), gid=%d (%s) "
+                                   "on host %s, pid %d\n",
+                                   os.getuid(), pwd.getpwuid(os.getuid())[0],
+                                   os.getgid(), grp.getgrgid(os.getgid())[0],
+                                   socket.gethostname(), os.getpid())
+                except ImportError:
+                    pass
+                err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
+        if _wsgi_server is None:
+            # Serve up the error with our own mini WSGI app.
+            headers = [ ('Content-type', 'text/plain'),
+                        ('Content-length', str(len(err))) ]
+            start_response("500 Internal Server Error", headers)
+            return [err]
+
+        # Call the normal application
+        return _wsgi_server.wsgi_application(environ, start_response)
+    return application

nilmdb/utils/__init__.py

@@ -1,9 +1,15 @@
 """NilmDB utilities"""
 
-from .timer import Timer
-from .iteratorizer import Iteratorizer
-from .serializer import Serializer
-from .lrucache import lru_cache
-from .diskusage import du
-from .mustclose import must_close
-from .urllib import urlencode
+from __future__ import absolute_import
+from nilmdb.utils.timer import Timer
+from nilmdb.utils.serializer import serializer_proxy
+from nilmdb.utils.lrucache import lru_cache
+from nilmdb.utils.diskusage import du, human_size
+from nilmdb.utils.mustclose import must_close
+from nilmdb.utils import atomic
+import nilmdb.utils.threadsafety
+import nilmdb.utils.fallocate
+import nilmdb.utils.time
+import nilmdb.utils.iterator
+import nilmdb.utils.interval
+import nilmdb.utils.lock

nilmdb/utils/atomic.py

@@ -0,0 +1,26 @@
+# Atomic file writing helper.
+
+import os
+
+def replace_file(filename, content):
+    """Attempt to atomically and durably replace the filename with the
+    given contents.  This is intended to be 'pretty good on most
+    OSes', but not necessarily bulletproof."""
+
+    newfilename = filename + ".new"
+
+    # Write to new file, flush it
+    with open(newfilename, "wb") as f:
+        f.write(content)
+        f.flush()
+        os.fsync(f.fileno())
+
+    # Move new file over old one
+    try:
+        os.rename(newfilename, filename)
+    except OSError: # pragma: no cover
+        # Some OSes might not support renaming over an existing file.
+        # This is definitely NOT atomic!
+        os.remove(filename)
+        os.rename(newfilename, filename)
+

nilmdb/utils/diskusage.py

@@ -1,8 +1,7 @@
-import nilmdb
 import os
 from math import log
 
-def sizeof_fmt(num):
+def human_size(num):
     """Human friendly file size"""
     unit_list = zip(['bytes', 'kiB', 'MiB', 'GiB', 'TiB'], [0, 0, 1, 2, 2])
     if num > 1:
@@ -16,15 +15,11 @@ def sizeof_fmt(num):
     if num == 1: # pragma: no cover
         return '1 byte'
 
-def du_bytes(path):
+def du(path):
     """Like du -sb, returns total size of path in bytes."""
     size = os.path.getsize(path)
     if os.path.isdir(path):
-        for file in os.listdir(path):
-            filepath = os.path.join(path, file)
-            size += du_bytes(filepath)
+        for thisfile in os.listdir(path):
+            filepath = os.path.join(path, thisfile)
+            size += du(filepath)
     return size
-
-def du(path):
-    """Like du -sh, returns total size of path as a human-readable string."""
-    return sizeof_fmt(du_bytes(path))

nilmdb/utils/fallocate.py

@@ -0,0 +1,49 @@
+# Implementation of hole punching via fallocate, if the OS
+# and filesystem support it.
+
+try:
+    import os
+    import ctypes
+    import ctypes.util
+
+    def make_fallocate():
+        libc_name = ctypes.util.find_library('c')
+        libc = ctypes.CDLL(libc_name, use_errno=True)
+
+        _fallocate = libc.fallocate
+        _fallocate.restype = ctypes.c_int
+        _fallocate.argtypes = [ ctypes.c_int, ctypes.c_int,
+                                ctypes.c_int64, ctypes.c_int64 ]
+
+        del libc
+        del libc_name
+
+        def fallocate(fd, mode, offset, len_):
+            res = _fallocate(fd, mode, offset, len_)
+            if res != 0: # pragma: no cover
+                errno = ctypes.get_errno()
+                raise IOError(errno, os.strerror(errno))
+        return fallocate
+
+    fallocate = make_fallocate()
+    del make_fallocate
+except Exception: # pragma: no cover
+    fallocate = None
+
+FALLOC_FL_KEEP_SIZE = 0x01
+FALLOC_FL_PUNCH_HOLE = 0x02
+
+def punch_hole(filename, offset, length, ignore_errors = True):
+    """Punch a hole in the file.  This isn't well supported, so errors
+    are ignored by default."""
+    try:
+        if fallocate is None: # pragma: no cover
+            raise IOError("fallocate not available")
+        with open(filename, "r+") as f:
+            fallocate(f.fileno(),
+                      FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+                      offset, length)
+    except IOError: # pragma: no cover
+        if ignore_errors:
+            return
+        raise

nilmdb/utils/interval.py

@@ -0,0 +1,106 @@
+"""Interval.  Like nilmdb.server.interval, but re-implemented here
+in plain Python so clients have easier access to it.
+
+Intervals are half-open, ie. they include data points with timestamps
+[start, end)
+"""
+
+import nilmdb.utils.time
+import nilmdb.utils.iterator
+
+class IntervalError(Exception):
+    """Error due to interval overlap, etc"""
+    pass
+
+# Interval
+class Interval:
+    """Represents an interval of time."""
+
+    def __init__(self, start, end):
+        """
+        'start' and 'end' are arbitrary numbers that represent time
+        """
+        if start >= end:
+            # Explicitly disallow zero-width intervals (since they're half-open)
+            raise IntervalError("start %s must precede end %s" % (start, end))
+        self.start = start
+        self.end = end
+
+    def __repr__(self):
+        s = repr(self.start) + ", " + repr(self.end)
+        return self.__class__.__name__ + "(" + s + ")"
+
+    def __str__(self):
+        return ("[" + nilmdb.utils.time.timestamp_to_string(self.start) +
+                " -> " + nilmdb.utils.time.timestamp_to_string(self.end) + ")")
+
+    def __cmp__(self, other):
+        """Compare two intervals.  If non-equal, order by start then end"""
+        return cmp(self.start, other.start) or cmp(self.end, other.end)
+
+    def intersects(self, other):
+        """Return True if two Interval objects intersect"""
+        if not isinstance(other, Interval):
+            raise TypeError("need an Interval")
+        if self.end <= other.start or self.start >= other.end:
+            return False
+        return True
+
+    def subset(self, start, end):
+        """Return a new Interval that is a subset of this one"""
+        # A subclass that tracks additional data might override this.
+        if start < self.start or end > self.end:
+            raise IntervalError("not a subset")
+        return Interval(start, end)
+
+def set_difference(a, b):
+    """
+    Compute the difference (a \\ b) between the intervals in 'a' and
+    the intervals in 'b'; i.e., the ranges that are present in 'self'
+    but not 'other'.
+
+    'a' and 'b' must both be iterables.
+
+    Returns a generator that yields each interval in turn.
+    Output intervals are built as subsets of the intervals in the
+    first argument (a).
+    """
+    # Iterate through all starts and ends in sorted order.  Add a
+    # tag to the iterator so that we can figure out which one they
+    # were, after sorting.
+    def decorate(it, key_start, key_end):
+        for i in it:
+            yield i.start, key_start, i
+            yield i.end, key_end, i
+    a_iter = decorate(iter(a), 0, 2)
+    b_iter = decorate(iter(b), 1, 3)
+
+    # Now iterate over the timestamps of each start and end.
+    # At each point, evaluate which type of end it is, to determine
+    # how to build up the output intervals.
+    a_interval = None
+    b_interval = None
+    out_start = None
+    for (ts, k, i) in nilmdb.utils.iterator.imerge(a_iter, b_iter):
+        if k == 0:
+            # start a interval
+            a_interval = i
+            if b_interval is None:
+                out_start = ts
+        elif k == 1:
+            # start b interval
+            b_interval = i
+            if out_start is not None and out_start != ts:
+                yield a_interval.subset(out_start, ts)
+            out_start = None
+        elif k == 2:
+            # end a interval
+            if out_start is not None and out_start != ts:
+                yield a_interval.subset(out_start, ts)
+            out_start = None
+            a_interval = None
+        elif k == 3:
+            # end b interval
+            b_interval = None
+            if a_interval:
+                out_start = ts

nilmdb/utils/iterator.py

@@ -0,0 +1,36 @@
+# Misc iterator tools
+
+# Iterator merging, based on http://code.activestate.com/recipes/491285/
+import heapq
+def imerge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Equivalent to:  sorted(itertools.chain(*iterables))
+
+    >>> list(imerge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    heappop, siftup, _Stop = heapq.heappop, heapq._siftup, StopIteration
+
+    h = []
+    h_append = h.append
+    for it in map(iter, iterables):
+        try:
+            next = it.next
+            h_append([next(), next])
+        except _Stop:
+            pass
+    heapq.heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, next = s = h[0]      # raises IndexError when h is empty
+                yield v
+                s[0] = next()           # raises StopIteration when exhausted
+                siftup(h, 0)            # restore heap condition
+        except _Stop:
+            heappop(h)                  # remove empty iterator
+        except IndexError:
+            return

nilmdb/utils/iteratorizer.py

@@ -1,72 +0,0 @@
-import Queue
-import threading
-import sys
-
-# This file provides a class that will convert a function that
-# takes a callback into a generator that returns an iterator.
-
-# Based partially on http://stackoverflow.com/questions/9968592/
-
-class IteratorizerThread(threading.Thread):
-    def __init__(self, queue, function):
-        """
-        function: function to execute, which takes the
-        callback (provided by this class) as an argument
-        """
-        threading.Thread.__init__(self)
-        self.function = function
-        self.queue = queue
-        self.die = False
-
-    def callback(self, data):
-        if self.die:
-            raise Exception("should die")
-        self.queue.put((1, data))
-
-    def run(self):
-        try:
-            result = self.function(self.callback)
-        except:
-            if sys is not None: # can be None during unclean shutdown
-                self.queue.put((2, sys.exc_info()))
-        else:
-            self.queue.put((0, result))
-
-class Iteratorizer(object):
-    def __init__(self, function):
-        """
-        function: function to execute, which takes the
-        callback (provided by this class) as an argument
-        """
-        self.function = function
-        self.queue = Queue.Queue(maxsize = 1)
-        self.thread = IteratorizerThread(self.queue, self.function)
-        self.thread.daemon = True
-        self.thread.start()
-
-    def __del__(self):
-        # If we get garbage collected, try to get rid of the
-        # thread too by asking it to raise an exception, then
-        # draining the queue until it's gone.
-        self.thread.die = True
-        while self.thread.isAlive():
-            try:
-                self.queue.get(True, 0.01)
-            except: # pragma: no cover
-                pass
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        (typ, data) = self.queue.get()
-        if typ == 0:
-            # function returned
-            self.retval = data
-            raise StopIteration
-        elif typ == 1:
-            # data available
-            return data
-        else:
-            # exception
-            raise data[0], data[1], data[2]

nilmdb/utils/lock.py

@@ -0,0 +1,33 @@
+# File locking
+
+import warnings
+
+try:
+    import fcntl
+    import errno
+
+    def exclusive_lock(f):
+        """Acquire an exclusive lock.  Returns True on successful
+        lock, or False on error."""
+        try:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError as e:
+            if e.errno in (errno.EACCES, errno.EAGAIN):
+                return False
+            else: # pragma: no cover
+                raise
+        return True
+
+    def exclusive_unlock(f):
+        """Release an exclusive lock."""
+        fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+
+except ImportError: # pragma: no cover
+    def exclusive_lock(f):
+        """Dummy lock function -- does not lock!"""
+        warnings.warn("Pretending to lock " + str(f))
+        return True
+
+    def exclusive_unlock(f):
+        """Release an exclusive lock."""
+        return

nilmdb/utils/lrucache.py

@@ -4,17 +4,18 @@
 # with added 'destructor' functionality.
 
 import collections
-import functools
+import decorator
 
-def lru_cache(size = 10, onremove = None):
+def lru_cache(size = 10, onremove = None, keys = slice(None)):
     """Least-recently-used cache decorator.
 
     @lru_cache(size = 10, onevict = None)
     def f(...):
         pass
 
-    Given a function and arguments, memoize its return value.
-    Up to 'size' elements are cached.
+    Given a function and arguments, memoize its return value.  Up to
+    'size' elements are cached.  'keys' is a slice object that
+    represents which arguments are used as the cache key.
 
     When evicting a value from the cache, call the function
     'onremove' with the value that's being evicted.
@@ -24,43 +25,52 @@ def lru_cache(size = 10, onremove = None):
     f.cache_hits and f.cache_misses give statistics.
     """
 
-    def decorator(func):
+    def decorate(func):
         cache = collections.OrderedDict()	# order: least- to most-recent
 
         def evict(value):
             if onremove:
                 onremove(value)
 
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            key = args + tuple(sorted(kwargs.items()))
+        def wrapper(orig, *args, **kwargs):
+            if kwargs:
+                raise NotImplementedError("kwargs not supported")
+            key = args[keys]
             try:
                 value = cache.pop(key)
-                wrapper.cache_hits += 1
+                orig.cache_hits += 1
             except KeyError:
-                value = func(*args, **kwargs)
-                wrapper.cache_misses += 1
+                value = orig(*args)
+                orig.cache_misses += 1
                 if len(cache) >= size:
                     evict(cache.popitem(0)[1])	# evict LRU cache entry
             cache[key] = value              	# (re-)insert this key at end
             return value
 
-        def cache_remove(*args, **kwargs):
-            """Remove the described key from this cache, if present.
-            Note that if the original wrapped function was implicitly
-            passed 'self', you need to pass it as an argument here too."""
-            key = args + tuple(sorted(kwargs.items()))
+        def cache_remove(*args):
+            """Remove the described key from this cache, if present."""
+            key = args
             if key in cache:
                 evict(cache.pop(key))
+            else:
+                if len(cache) > 0 and len(args) != len(cache.iterkeys().next()):
+                    raise KeyError("trying to remove from LRU cache, but "
+                                   "number of arguments doesn't match the "
+                                   "cache key length")
 
         def cache_remove_all():
             for key in cache:
                 evict(cache.pop(key))
 
-        wrapper.cache_hits = 0
-        wrapper.cache_misses = 0
-        wrapper.cache_remove = cache_remove
-        wrapper.cache_remove_all = cache_remove_all
+        def cache_info():
+            return (func.cache_hits, func.cache_misses)
 
-        return wrapper
-    return decorator
+        new = decorator.decorator(wrapper, func)
+        func.cache_hits = 0
+        func.cache_misses = 0
+        new.cache_info = cache_info
+        new.cache_remove = cache_remove
+        new.cache_remove_all = cache_remove_all
+        return new
+
+    return decorate

nilmdb/utils/mustclose.py

@@ -1,42 +1,61 @@
-# Class decorator that warns on stderr at deletion time if the class's
-# close() member wasn't called.
-
 from nilmdb.utils.printf import *
 import sys
+import inspect
+import decorator
 
-def must_close(errorfile = sys.stderr):
-    def decorator(cls):
-        def dummy(*args, **kwargs):
-            pass
-        if "__init__" not in cls.__dict__:
-            cls.__init__ = dummy
-        if "__del__" not in cls.__dict__:
-            cls.__del__ = dummy
-        if "close" not in cls.__dict__:
-            cls.close = dummy
+def must_close(errorfile = sys.stderr, wrap_verify = False):
+    """Class decorator that warns on 'errorfile' at deletion time if
+    the class's close() member wasn't called.
 
-        orig_init = cls.__init__
-        orig_del = cls.__del__
-        orig_close = cls.close
+    If 'wrap_verify' is True, every class method is wrapped with a
+    verifier that will raise AssertionError if the .close() method has
+    already been called."""
+    def class_decorator(cls):
 
-        def __init__(self, *args, **kwargs):
-            ret = orig_init(self, *args, **kwargs)
+        def wrap_class_method(wrapper):
+            try:
+                orig = getattr(cls, wrapper.__name__).im_func
+            except Exception:
+                orig = lambda x: None
+            setattr(cls, wrapper.__name__, decorator.decorator(wrapper, orig))
+
+        @wrap_class_method
+        def __init__(orig, self, *args, **kwargs):
+            ret = orig(self, *args, **kwargs)
             self.__dict__["_must_close"] = True
+            self.__dict__["_must_close_initialized"] = True
             return ret
 
-        def __del__(self):
+        @wrap_class_method
+        def __del__(orig, self, *args, **kwargs):
             if "_must_close" in self.__dict__:
                 fprintf(errorfile, "error: %s.close() wasn't called!\n",
                         self.__class__.__name__)
-            return orig_del(self)
+            return orig(self, *args, **kwargs)
 
-        def close(self, *args, **kwargs):
+        @wrap_class_method
+        def close(orig, self, *args, **kwargs):
+            if "_must_close" in self.__dict__:
                 del self._must_close
-            return orig_close(self)
+            return orig(self, *args, **kwargs)
 
-        cls.__init__ = __init__
-        cls.__del__ = __del__
-        cls.close = close
+        # Optionally wrap all other functions
+        def verifier(orig, self, *args, **kwargs):
+            if ("_must_close" not in self.__dict__ and
+                "_must_close_initialized" in self.__dict__):
+                raise AssertionError("called " + str(orig) + " after close")
+            return orig(self, *args, **kwargs)
+        if wrap_verify:
+            for (name, method) in inspect.getmembers(cls, inspect.ismethod):
+                # Skip class methods
+                if method.__self__ is not None:
+                    continue
+                # Skip some methods
+                if name in [ "__del__", "__init__" ]:
+                    continue
+                # Set up wrapper
+                setattr(cls, name, decorator.decorator(verifier,
+                                                       method.im_func))
 
         return cls
-    return decorator
+    return class_decorator

nilmdb/utils/serializer.py

@@ -1,6 +1,10 @@
 import Queue
 import threading
 import sys
+import decorator
+import inspect
+import types
+import functools
 
 # This file provides a class that will wrap an object and serialize
 # all calls to its methods.  All calls to that object will be queued
@@ -12,8 +16,9 @@ import sys
 class SerializerThread(threading.Thread):
     """Thread that retrieves call information from the queue, makes the
     call, and returns the results."""
-    def __init__(self, call_queue):
+    def __init__(self, classname, call_queue):
         threading.Thread.__init__(self)
+        self.name = "Serializer-" + classname + "-" + self.name
         self.call_queue = call_queue
 
     def run(self):
@@ -22,51 +27,83 @@ class SerializerThread(threading.Thread):
             # Terminate if result_queue is None
             if result_queue is None:
                 return
+            exception = None
+            result = None
             try:
                 result = func(*args, **kwargs) # wrapped
             except:
-                result_queue.put((sys.exc_info(), None))
-            else:
-                result_queue.put((None, result))
+                exception = sys.exc_info()
+            # Ensure we delete these before returning a result, so
+            # we don't unncessarily hold onto a reference while
+            # we're waiting for the next call.
+            del func, args, kwargs
+            result_queue.put((exception, result))
+            del exception, result
 
-class WrapCall(object):
-    """Wrap a callable using the given queues"""
+def serializer_proxy(obj_or_type):
+    """Wrap the given object or type in a SerializerObjectProxy.
 
-    def __init__(self, call_queue, result_queue, func):
+    Returns a SerializerObjectProxy object that proxies all method
+    calls to the object, as well as attribute retrievals.
+
+    The proxied requests, including instantiation, are performed in a
+    single thread and serialized between caller threads.
+    """
+    class SerializerCallProxy(object):
+        def __init__(self, call_queue, func, objectproxy):
             self.call_queue = call_queue
-        self.result_queue = result_queue
             self.func = func
-
+            # Need to hold a reference to object proxy so it doesn't
+            # go away (and kill the thread) until after get called.
+            self.objectproxy = objectproxy
         def __call__(self, *args, **kwargs):
-        self.call_queue.put((self.result_queue, self.func, args, kwargs))
-        ( exc_info, result ) = self.result_queue.get()
+            result_queue = Queue.Queue()
+            self.call_queue.put((result_queue, self.func, args, kwargs))
+            ( exc_info, result ) = result_queue.get()
             if exc_info is None:
                 return result
             else:
                 raise exc_info[0], exc_info[1], exc_info[2]
 
-class WrapObject(object):
-    """Wrap all calls to methods in a target object with WrapCall"""
-
-    def __init__(self, target):
-        self.__wrap_target = target
-        self.__wrap_call_queue = Queue.Queue()
-        self.__wrap_serializer = SerializerThread(self.__wrap_call_queue)
-        self.__wrap_serializer.daemon = True
-        self.__wrap_serializer.start()
+    class SerializerObjectProxy(object):
+        def __init__(self, obj_or_type, *args, **kwargs):
+            self.__object = obj_or_type
+            try:
+                if type(obj_or_type) in (types.TypeType, types.ClassType):
+                    classname = obj_or_type.__name__
+                else:
+                    classname = obj_or_type.__class__.__name__
+            except AttributeError: # pragma: no cover
+                classname = "???"
+            self.__call_queue = Queue.Queue()
+            self.__thread = SerializerThread(classname, self.__call_queue)
+            self.__thread.daemon = True
+            self.__thread.start()
+            self._thread_safe = True
 
         def __getattr__(self, key):
-        """Wrap methods of self.__wrap_target in a WrapCall instance"""
-        func = getattr(self.__wrap_target, key)
-        if not callable(func):
-            raise TypeError("Can't serialize attribute %r (type: %s)"
-                            % (key, type(func)))
-        result_queue = Queue.Queue()
-        return WrapCall(self.__wrap_call_queue, result_queue, func)
+            if key.startswith("_SerializerObjectProxy__"): # pragma: no cover
+                raise AttributeError
+            attr = getattr(self.__object, key)
+            if not callable(attr):
+                getter = SerializerCallProxy(self.__call_queue, getattr, self)
+                return getter(self.__object, key)
+            r = SerializerCallProxy(self.__call_queue, attr, self)
+            return r
+
+        def __call__(self, *args, **kwargs):
+            """Call this to instantiate the type, if a type was passed
+            to serializer_proxy.  Otherwise, pass the call through."""
+            ret = SerializerCallProxy(self.__call_queue,
+                                      self.__object, self)(*args, **kwargs)
+            if type(self.__object) in (types.TypeType, types.ClassType):
+                # Instantiation
+                self.__object = ret
+                return self
+            return ret
 
         def __del__(self):
-        self.__wrap_call_queue.put((None, None, None, None))
-        self.__wrap_serializer.join()
+            self.__call_queue.put((None, None, None, None))
+            self.__thread.join()
 
-# Just an alias
-Serializer = WrapObject
+    return SerializerObjectProxy(obj_or_type)

nilmdb/utils/threadsafety.py

@@ -0,0 +1,109 @@
+from nilmdb.utils.printf import *
+import threading
+import warnings
+import types
+
+def verify_proxy(obj_or_type, exception = False, check_thread = True,
+                 check_concurrent = True):
+    """Wrap the given object or type in a VerifyObjectProxy.
+
+    Returns a VerifyObjectProxy that proxies all method calls to the
+    given object, as well as attribute retrievals.
+
+    When calling methods, the following checks are performed.  If
+    exception is True, an exception is raised.  Otherwise, a warning
+    is printed.
+
+    check_thread = True     # Warn/fail if two different threads call methods.
+    check_concurrent = True # Warn/fail if two functions are concurrently
+                            # run through this proxy
+    """
+    class Namespace(object):
+        pass
+    class VerifyCallProxy(object):
+        def __init__(self, func, parent_namespace):
+            self.func = func
+            self.parent_namespace = parent_namespace
+
+        def __call__(self, *args, **kwargs):
+            p = self.parent_namespace
+            this = threading.current_thread()
+            try:
+                callee = self.func.__name__
+            except AttributeError:
+                callee = "???"
+
+            if p.thread is None:
+                p.thread = this
+                p.thread_callee = callee
+
+            if check_thread and p.thread != this:
+                err = sprintf("unsafe threading: %s called %s.%s,"
+                              " but %s called %s.%s",
+                              p.thread.name, p.classname, p.thread_callee,
+                              this.name, p.classname, callee)
+                if exception:
+                    raise AssertionError(err)
+                else: # pragma: no cover
+                    warnings.warn(err)
+
+            need_concur_unlock = False
+            if check_concurrent:
+                if p.concur_lock.acquire(False) == False:
+                    err = sprintf("unsafe concurrency: %s called %s.%s "
+                                  "while %s is still in %s.%s",
+                                  this.name, p.classname, callee,
+                                  p.concur_tname, p.classname, p.concur_callee)
+                    if exception:
+                        raise AssertionError(err)
+                    else: # pragma: no cover
+                        warnings.warn(err)
+                else:
+                    p.concur_tname = this.name
+                    p.concur_callee = callee
+                    need_concur_unlock = True
+
+            try:
+                ret = self.func(*args, **kwargs)
+            finally:
+                if need_concur_unlock:
+                    p.concur_lock.release()
+            return ret
+
+    class VerifyObjectProxy(object):
+        def __init__(self, obj_or_type, *args, **kwargs):
+            p = Namespace()
+            self.__ns = p
+            p.thread = None
+            p.thread_callee = None
+            p.concur_lock = threading.Lock()
+            p.concur_tname = None
+            p.concur_callee = None
+            self.__obj = obj_or_type
+            try:
+                if type(obj_or_type) in (types.TypeType, types.ClassType):
+                    p.classname = self.__obj.__name__
+                else:
+                    p.classname = self.__obj.__class__.__name__
+            except AttributeError: # pragma: no cover
+                p.classname = "???"
+
+        def __getattr__(self, key):
+            if key.startswith("_VerifyObjectProxy__"): # pragma: no cover
+                raise AttributeError
+            attr = getattr(self.__obj, key)
+            if not callable(attr):
+                return VerifyCallProxy(getattr, self.__ns)(self.__obj, key)
+            return VerifyCallProxy(attr, self.__ns)
+
+        def __call__(self, *args, **kwargs):
+            """Call this to instantiate the type, if a type was passed
+            to verify_proxy.  Otherwise, pass the call through."""
+            ret = VerifyCallProxy(self.__obj, self.__ns)(*args, **kwargs)
+            if type(self.__obj) in (types.TypeType, types.ClassType):
+                # Instantiation
+                self.__obj = ret
+                return self
+            return ret
+
+    return VerifyObjectProxy(obj_or_type)

nilmdb/utils/time.py

@@ -0,0 +1,130 @@
+from __future__ import absolute_import
+
+from nilmdb.utils import datetime_tz
+import re
+import time
+
+# Range
+min_timestamp = (-2**63)
+max_timestamp = (2**62 - 1)
+
+# Smallest representable step
+epsilon = 1
+
+def string_to_timestamp(str):
+    """Convert a string that represents an integer number of microseconds
+    since epoch."""
+    try:
+        # Parse a string like "1234567890123456" and return an integer
+        return int(str)
+    except ValueError:
+        # Try parsing as a float, in case it's "1234567890123456.0"
+        return int(round(float(str)))
+
+def timestamp_to_string(timestamp):
+    """Convert a timestamp (integer microseconds since epoch) to a string"""
+    if isinstance(timestamp, float):
+        return str(int(round(timestamp)))
+    else:
+        return str(timestamp)
+
+def timestamp_to_human(timestamp):
+    """Convert a timestamp (integer microseconds since epoch) to a
+    human-readable string, using the local timezone for display
+    (e.g. from the TZ env var)."""
+    dt = datetime_tz.datetime_tz.fromtimestamp(timestamp_to_unix(timestamp))
+    return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
+
+def unix_to_timestamp(unix):
+    """Convert a Unix timestamp (floating point seconds since epoch)
+    into a NILM timestamp (integer microseconds since epoch)"""
+    return int(round(unix * 1e6))
+seconds_to_timestamp = unix_to_timestamp
+
+def timestamp_to_unix(timestamp):
+    """Convert a NILM timestamp (integer microseconds since epoch)
+    into a Unix timestamp (floating point seconds since epoch)"""
+    return timestamp / 1e6
+timestamp_to_seconds = timestamp_to_unix
+
+def rate_to_period(hz, cycles = 1):
+    """Convert a rate (in Hz) to a period (in timestamp units).
+    Returns an integer."""
+    period = unix_to_timestamp(cycles) / float(hz)
+    return int(round(period))
+
+def parse_time(toparse):
+    """
+    Parse a free-form time string and return a nilmdb timestamp
+    (integer seconds since epoch).  If the string doesn't contain a
+    timestamp, the current local timezone is assumed (e.g. from the TZ
+    env var).
+    """
+    if toparse == "min":
+        return min_timestamp
+    if toparse == "max":
+        return max_timestamp
+
+    # If string isn't "now" and doesn't contain at least 4 digits,
+    # consider it invalid.  smartparse might otherwise accept
+    # empty strings and strings with just separators.
+    if toparse != "now" and len(re.findall(r"\d", toparse)) < 4:
+        raise ValueError("not enough digits for a timestamp")
+
+    # Try to just parse the time as given
+    try:
+        return unix_to_timestamp(datetime_tz.datetime_tz.
+                                 smartparse(toparse).totimestamp())
+    except (ValueError, OverflowError):
+        pass
+
+    # If it starts with @, treat it as a NILM timestamp
+    # (integer microseconds since epoch)
+    try:
+        if toparse[0] == '@':
+            return int(toparse[1:])
+    except (ValueError, KeyError):
+        pass
+
+    # If it's parseable as a float, treat it as a Unix or NILM
+    # timestamp based on its range.
+    try:
+        val = float(toparse)
+        # range is from about year 2001 - 2128
+        if val > 1e9 and val < 5e9:
+            return unix_to_timestamp(val)
+        if val > 1e15 and val < 5e15:
+            return val
+    except ValueError:
+        pass
+
+    # Try to extract a substring in a condensed format that we expect
+    # to see in a filename or header comment
+    res = re.search(r"(^|[^\d])("            # non-numeric or SOL
+                    r"(199\d|2\d\d\d)"       # year
+                    r"[-/]?"                 # separator
+                    r"(0[1-9]|1[012])"       # month
+                    r"[-/]?"                 # separator
+                    r"([012]\d|3[01])"       # day
+                    r"[-T ]?"                # separator
+                    r"([01]\d|2[0-3])"       # hour
+                    r"[:]?"                  # separator
+                    r"([0-5]\d)"             # minute
+                    r"[:]?"                  # separator
+                    r"([0-5]\d)?"            # second
+                    r"([-+]\d\d\d\d)?"       # timezone
+                    r")", toparse)
+    if res is not None:
+        try:
+            return unix_to_timestamp(datetime_tz.datetime_tz.
+                                     smartparse(res.group(2)).totimestamp())
+        except ValueError:
+            pass
+
+    # Could also try to successively parse substrings, but let's
+    # just give up for now.
+    raise ValueError("unable to parse timestamp")
+
+def now():
+    """Return current timestamp"""
+    return unix_to_timestamp(time.time())

nilmdb/utils/timer.py

@@ -2,10 +2,11 @@
 
 # Simple timer to time a block of code, for optimization debugging
 # use like:
-#   with nilmdb.Timer("flush"):
+#   with nilmdb.utils.Timer("flush"):
 #       foo.flush()
 
 from __future__ import print_function
+from __future__ import absolute_import
 import contextlib
 import time
 

nilmdb/utils/timestamper.py

@@ -1,22 +1,18 @@
 """File-like objects that add timestamps to the input lines"""
 
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
-
-import time
-import os
-import datetime_tz
+import nilmdb.utils.time
 
 class Timestamper(object):
     """A file-like object that adds timestamps to lines of an input file."""
-    def __init__(self, file, ts_iter):
+    def __init__(self, infile, ts_iter):
         """file: filename, or another file-like object
            ts_iter: iterator that returns a timestamp string for
            each line of the file"""
-        if isinstance(file, basestring):
-            self.file = open(file, "r")
+        if isinstance(infile, basestring):
+            self.file = open(infile, "r")
         else:
-            self.file = file
+            self.file = infile
         self.ts_iter = ts_iter
 
     def close(self):
@@ -55,7 +51,7 @@ class Timestamper(object):
 
 class TimestamperRate(Timestamper):
     """Timestamper that uses a start time and a fixed rate"""
-    def __init__(self, file, start, rate, end = None):
+    def __init__(self, infile, start, rate, end = None):
         """
         file: file name or object
 
@@ -65,44 +61,33 @@ class TimestamperRate(Timestamper):
 
         end: If specified, raise StopIteration before outputting a value
              greater than this."""
+        timestamp_to_string = nilmdb.utils.time.timestamp_to_string
+        rate_to_period = nilmdb.utils.time.rate_to_period
         def iterator(start, rate, end):
             n = 0
             rate = float(rate)
             while True:
-                now = start + n / rate
+                now = start + rate_to_period(rate, n)
                 if end and now >= end:
                     raise StopIteration
-                yield sprintf("%.6f ", start + n / rate)
+                yield timestamp_to_string(now) + " "
                 n += 1
-        # Handle case where we're passed a datetime or datetime_tz object
-        if "totimestamp" in dir(start):
-            start = start.totimestamp()
-        Timestamper.__init__(self, file, iterator(start, rate, end))
+        Timestamper.__init__(self, infile, iterator(start, rate, end))
         self.start = start
         self.rate = rate
     def __str__(self):
-        start = datetime_tz.datetime_tz.fromtimestamp(self.start)
-        start = start.strftime("%a, %d %b %Y %H:%M:%S %Z")
         return sprintf("TimestamperRate(..., start=\"%s\", rate=%g)",
-                       str(start), self.rate)
+                       nilmdb.utils.time.timestamp_to_human(self.start),
+                       self.rate)
 
 class TimestamperNow(Timestamper):
     """Timestamper that uses current time"""
-    def __init__(self, file):
+    def __init__(self, infile):
+        timestamp_to_string = nilmdb.utils.time.timestamp_to_string
+        get_now = nilmdb.utils.time.now
         def iterator():
             while True:
-                now = datetime_tz.datetime_tz.utcnow().totimestamp()
-                yield sprintf("%.6f ", now)
-        Timestamper.__init__(self, file, iterator())
+                yield timestamp_to_string(get_now()) + " "
+        Timestamper.__init__(self, infile, iterator())
     def __str__(self):
         return "TimestamperNow(...)"
-
-class TimestamperNull(Timestamper):
-    """Timestamper that adds nothing to each line"""
-    def __init__(self, file):
-        def iterator():
-            while True:
-                yield ""
-        Timestamper.__init__(self, file, iterator())
-    def __str__(self):
-        return "TimestamperNull(...)"

nilmdb/utils/urllib.py

@@ -1,68 +0,0 @@
-from __future__ import absolute_import
-from urllib import quote_plus, _is_unicode
-
-# urllib.urlencode insists on encoding Unicode as ASCII.  This is an
-# exact copy of that function, except we encode it as UTF-8 instead.
-
-def urlencode(query, doseq=0):
-    """Encode a sequence of two-element tuples or dictionary into a URL query string.
-
-    If any values in the query arg are sequences and doseq is true, each
-    sequence element is converted to a separate parameter.
-
-    If the query arg is a sequence of two-element tuples, the order of the
-    parameters in the output will match the order of parameters in the
-    input.
-    """
-
-    if hasattr(query,"items"):
-        # mapping objects
-        query = query.items()
-    else:
-        # it's a bother at times that strings and string-like objects are
-        # sequences...
-        try:
-            # non-sequence items should not work with len()
-            # non-empty strings will fail this
-            if len(query) and not isinstance(query[0], tuple):
-                raise TypeError
-            # zero-length sequences of all types will get here and succeed,
-            # but that's a minor nit - since the original implementation
-            # allowed empty dicts that type of behavior probably should be
-            # preserved for consistency
-        except TypeError:
-            ty,va,tb = sys.exc_info()
-            raise TypeError, "not a valid non-string sequence or mapping object", tb
-
-    l = []
-    if not doseq:
-        # preserve old behavior
-        for k, v in query:
-            k = quote_plus(str(k))
-            v = quote_plus(str(v))
-            l.append(k + '=' + v)
-    else:
-        for k, v in query:
-            k = quote_plus(str(k))
-            if isinstance(v, str):
-                v = quote_plus(v)
-                l.append(k + '=' + v)
-            elif _is_unicode(v):
-                # is there a reasonable way to convert to ASCII?
-                # encode generates a string, but "replace" or "ignore"
-                # lose information and "strict" can raise UnicodeError
-                v = quote_plus(v.encode("utf-8","strict"))
-                l.append(k + '=' + v)
-            else:
-                try:
-                    # is this a sufficient test for sequence-ness?
-                    len(v)
-                except TypeError:
-                    # not a sequence
-                    v = quote_plus(str(v))
-                    l.append(k + '=' + v)
-                else:
-                    # loop over the sequence
-                    for elt in v:
-                        l.append(k + '=' + quote_plus(str(elt)))
-    return '&'.join(l)

nilmtool.py

@@ -1,6 +0,0 @@
-#!/usr/bin/python
-
-import nilmdb
-import sys
-
-nilmdb.cmdline.Cmdline(sys.argv[1:]).run()

runserver.py

@@ -1,35 +0,0 @@
-#!/usr/bin/python
-
-import nilmdb
-import argparse
-
-formatter = argparse.ArgumentDefaultsHelpFormatter
-parser = argparse.ArgumentParser(description='Run the NILM server',
-                                 formatter_class = formatter)
-parser.add_argument('-p', '--port', help='Port number', type=int, default=12380)
-parser.add_argument('-d', '--database', help='Database directory', default="db")
-parser.add_argument('-y', '--yappi', help='Run with yappi profiler',
-                    action='store_true')
-args = parser.parse_args()
-
-# Start web app on a custom port
-db = nilmdb.NilmDB(args.database)
-server = nilmdb.Server(db, host = "127.0.0.1",
-                       port = args.port,
-                       embedded = False)
-
-
-if args.yappi:
-    print "Running in yappi"
-    try:
-        import yappi
-        yappi.start()
-        server.start(blocking = True)
-    finally:
-        yappi.stop()
-        print "Try: yappi.print_stats(sort_type=yappi.SORTTYPE_TTOT,limit=50)"
-        from IPython import embed
-        embed()
-else:
-    server.start(blocking = True)
-db.close()

setup.cfg

@@ -1,15 +1,27 @@
+[aliases]
+test = nosetests
+
 [nosetests]
-# note: the value doesn't matter, that's why they're empty here
-nocapture=
-nologcapture=        # comment to see cherrypy logs on failure
-with-coverage=
-cover-inclusive=
+# Note: values must be set to 1, and have no comments on the same line,
+# for "python setup.py nosetests" to work correctly.
+nocapture=1
+# Comment this out to see CherryPy logs on failure:
+nologcapture=1
+with-coverage=1
+cover-inclusive=1
 cover-package=nilmdb
-cover-erase=
-##cover-html=         # this works, puts html output in cover/ dir
-##cover-branches=     # need nose 1.1.3 for this
-stop=
+cover-erase=1
+# this works, puts html output in cover/ dir:
+# cover-html=1
+# need nose 1.1.3 for this:
+# cover-branches=1
+#debug=nose
+#debug-log=nose.log
+stop=1
 verbosity=2
+tests=tests
+#tests=tests/test_threadsafety.py
+#tests=tests/test_bulkdata.py
 #tests=tests/test_mustclose.py
 #tests=tests/test_lrucache.py
 #tests=tests/test_cmdline.py
@@ -23,6 +35,7 @@ verbosity=2
 #tests=tests/test_serializer.py
 #tests=tests/test_iteratorizer.py
 #tests=tests/test_client.py:TestClient.test_client_nilmdb
-#with-profile=
+#tests=tests/test_nilmdb.py
+#with-profile=1
 #profile-sort=time
 ##profile-restrict=10  # doesn't work right, treated as string or something

setup.py

@@ -0,0 +1,138 @@
+#!/usr/bin/python
+
+# To release a new version, tag it:
+#   git tag -a nilmdb-1.1 -m "Version 1.1"
+#   git push --tags
+# Then just package it up:
+#   python setup.py sdist
+
+# This is supposed to be using Distribute:
+#
+#   distutils provides a "setup" method.
+#   setuptools is a set of monkeypatches on top of that.
+#   distribute is a particular version/implementation of setuptools.
+#
+# So we don't really know if this is using the old setuptools or the
+# Distribute-provided version of setuptools.
+
+import traceback
+import sys
+import os
+
+try:
+    from setuptools import setup, find_packages
+    from distutils.extension import Extension
+    import distutils.version
+except ImportError:
+    traceback.print_exc()
+    print "Please install the prerequisites listed in README.txt"
+    sys.exit(1)
+
+# Versioneer manages version numbers from git tags.
+# https://github.com/warner/python-versioneer
+import versioneer
+versioneer.versionfile_source = 'nilmdb/_version.py'
+versioneer.versionfile_build = 'nilmdb/_version.py'
+versioneer.tag_prefix = 'nilmdb-'
+versioneer.parentdir_prefix = 'nilmdb-'
+
+# Hack to workaround logging/multiprocessing issue:
+# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
+try: import multiprocessing
+except Exception: pass
+
+# Use Cython if it's new enough, otherwise use preexisting C files.
+cython_modules = [ 'nilmdb.server.interval',
+                   'nilmdb.server.rbtree' ]
+try:
+    import Cython
+    from Cython.Build import cythonize
+    if (distutils.version.LooseVersion(Cython.__version__) <
+        distutils.version.LooseVersion("0.16")):
+        print "Cython version", Cython.__version__, "is too old; not using it."
+        raise ImportError()
+    use_cython = True
+except ImportError:
+    use_cython = False
+
+ext_modules = [ Extension('nilmdb.server.rocket', ['nilmdb/server/rocket.c' ]) ]
+for modulename in cython_modules:
+    filename = modulename.replace('.','/')
+    if use_cython:
+        ext_modules.extend(cythonize(filename + ".pyx"))
+    else:
+        cfile = filename + ".c"
+        if not os.path.exists(cfile):
+            raise Exception("Missing source file " + cfile + ".  "
+                            "Try installing cython >= 0.16.")
+        ext_modules.append(Extension(modulename, [ cfile ]))
+
+# We need a MANIFEST.in.  Generate it here rather than polluting the
+# repository with yet another setup-related file.
+with open("MANIFEST.in", "w") as m:
+    m.write("""
+# Root
+include README.txt
+include setup.cfg
+include setup.py
+include versioneer.py
+include Makefile
+include .coveragerc
+include .pylintrc
+
+# Cython files -- include source.
+recursive-include nilmdb/server *.pyx *.pyxdep *.pxd
+
+# Tests
+recursive-include tests *.py
+recursive-include tests/data *
+include tests/test.order
+
+# Docs
+recursive-include docs Makefile *.md
+
+# Extras
+recursive-include extras *
+""")
+
+# Run setup
+setup(name='nilmdb',
+      version = versioneer.get_version(),
+      cmdclass = versioneer.get_cmdclass(),
+      url = 'https://git.jim.sh/jim/lees/nilmdb.git',
+      author = 'Jim Paris',
+      description = "NILM Database",
+      long_description = "NILM Database",
+      license = "Proprietary",
+      author_email = 'jim@jtan.com',
+      tests_require = [ 'nose',
+                        'coverage',
+                        ],
+      setup_requires = [ 'distribute',
+                         ],
+      install_requires = [ 'decorator',
+                           'cherrypy >= 3.2',
+                           'simplejson',
+                           'pycurl',
+                           'python-dateutil',
+                           'pytz',
+                           'psutil >= 0.3.0',
+                           'requests >= 1.1.0, < 2.0.0',
+                           ],
+      packages = [ 'nilmdb',
+                   'nilmdb.utils',
+                   'nilmdb.utils.datetime_tz',
+                   'nilmdb.server',
+                   'nilmdb.client',
+                   'nilmdb.cmdline',
+                   'nilmdb.scripts',
+                   ],
+      entry_points = {
+          'console_scripts': [
+              'nilmtool = nilmdb.scripts.nilmtool:main',
+              'nilmdb-server = nilmdb.scripts.nilmdb_server:main',
+              ],
+          },
+      ext_modules = ext_modules,
+      zip_safe = False,
+      )

tests/aplotter.py

@@ -1,419 +0,0 @@
-
-#-----------------------------------------------
-#aplotter.py - ascii art function plotter
-#Copyright (c) 2006, Imri Goldberg
-#All rights reserved.
-#
-#Redistribution and use in source and binary forms,
-#with or without modification, are permitted provided
-#that the following conditions are met:
-#
-#    * Redistributions of source code must retain the
-#		above copyright notice, this list of conditions
-#		and the following disclaimer.
-#    * Redistributions in binary form must reproduce the
-#		above copyright notice, this list of conditions
-#		and the following disclaimer in the documentation
-#		and/or other materials provided with the distribution.
-#    * Neither the name of the <ORGANIZATION> nor the names of
-#		its contributors may be used to endorse or promote products
-#		derived from this software without specific prior written permission.
-#
-#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-#AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-#IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-#ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-#LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-#DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-#SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-#CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-#OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#-----------------------------------------------
-
-import math
-
-
-EPSILON = 0.000001
-
-def transposed(mat):
-	result = []
-	for i in xrange(len(mat[0])):
-		result.append([x[i] for x in mat])
-	return result
-
-def y_reversed(mat):    
-	result = []
-	for i in range(len(mat)):
-		result.append(list(reversed(mat[i])))
-	return result
-
-def sign(x):
-	if 0<x:
-		return 1
-	if 0 == x:
-		return 0
-	return -1
-
-class Plotter(object):
-
-	class PlotData(object):
-		def __init__(self, x_size, y_size, min_x, max_x, min_y, max_y, x_mod, y_mod):
-			self.x_size = x_size
-			self.y_size = y_size
-			self.min_x = min_x
-			self.max_x = max_x
-			self.min_y = min_y
-			self.max_y = max_y
-			self.x_mod = x_mod
-			self.y_mod = y_mod
-
-			self.x_step = float(max_x - min_x)/float(self.x_size)
-			self.y_step = float(max_y - min_y)/float(self.y_size)
-			self.inv_x_step = 1/self.x_step
-			self.inv_y_step = 1/self.y_step
-
-			self.ratio = self.y_step / self.x_step
-		def __repr__(self):
-			s = "size: %s, bl: %s, tr: %s, step: %s" % ((self.x_size, self.y_size), (self.min_x, self.min_y), (self.max_x, self.max_y),
-														 (self.x_step, self.y_step))
-			return s
-	
-	def __init__(self, **kwargs):
-
-		self.x_size = kwargs.get("x_size", 80)
-		self.y_size = kwargs.get("y_size", 20)
-
-		self.will_draw_axes = kwargs.get("draw_axes", True)
-
-		self.new_line = kwargs.get("newline", "\n")
-
-		self.dot = kwargs.get("dot", "*")
-
-		self.plot_slope = kwargs.get("plot_slope", True)
-
-		self.x_margin = kwargs.get("x_margin", 0.05)
-		self.y_margin = kwargs.get("y_margin", 0.1)
-
-		self.will_plot_labels = kwargs.get("plot_labels", True)
-
-	@staticmethod
-	def get_symbol_by_slope(slope, default_symbol):
-		draw_symbol = default_symbol
-		if slope > math.tan(3*math.pi/8):
-			draw_symbol = "|"
-		elif slope > math.tan(math.pi/8) and slope < math.tan(3*math.pi/8):
-			draw_symbol = "/"
-		elif abs(slope) < math.tan(math.pi/8):
-			draw_symbol = "-"
-		elif slope < math.tan(-math.pi/8) and slope > math.tan(-3*math.pi/8):
-			draw_symbol = "\\"
-		elif slope < math.tan(-3*math.pi/8):
-			draw_symbol = "|"
-		return draw_symbol    
-
-
-	def plot_labels(self, output_buffer, plot_data):
-		if plot_data.y_size < 2:
-			return
-
-		margin_factor = 1
-
-		do_plot_x_label = True
-		do_plot_y_label = True
-
-		x_str = "%+g"
-		if plot_data.x_size < 16:
-			do_plot_x_label = False
-		elif plot_data.x_size < 23:
-			x_str = "%+.2g" 
-
-		y_str = "%+g"    
-		if plot_data.x_size < 8:
-			do_plot_y_label = False
-		elif plot_data.x_size < 11:
-			y_str = "%+.2g"
-			
-		act_min_x = (plot_data.min_x + plot_data.x_mod*margin_factor)
-		act_max_x = (plot_data.max_x - plot_data.x_mod*margin_factor)
-		act_min_y = (plot_data.min_y + plot_data.y_mod*margin_factor)
-		act_max_y = (plot_data.max_y - plot_data.y_mod*margin_factor)
-
-		if abs(act_min_x) < 1:
-			min_x_str = "%+.2g" % act_min_x
-		else:
-			min_x_str = x_str % act_min_x
-
-		if abs(act_max_x) < 1:
-			max_x_str = "%+.2g" % act_max_x
-		else:
-			max_x_str = x_str % act_max_x
-		
-		if abs(act_min_y) < 1:
-			min_y_str = "%+.2g" % act_min_y
-		else:
-			min_y_str = y_str % act_min_y
-
-		if abs(act_max_y) < 1:
-			max_y_str = "%+.2g" % act_max_y
-		else:
-			max_y_str = y_str % act_max_y
-						 
-		min_x_coord = self.get_coord(act_min_x,plot_data.min_x,plot_data.x_step)
-		max_x_coord = self.get_coord(act_max_x,plot_data.min_x,plot_data.x_step)
-		min_y_coord = self.get_coord(act_min_y,plot_data.min_y,plot_data.y_step)
-		max_y_coord = self.get_coord(act_max_y,plot_data.min_y,plot_data.y_step)
-								 
-
-		#print plot_data
-		
-		y_zero_coord = self.get_coord(0, plot_data.min_y, plot_data.y_step)
-
-		#if plot_data.min_x < 0 and plot_data.max_x > 0:
-		x_zero_coord = self.get_coord(0, plot_data.min_x, plot_data.x_step)
-		#else:
-		
-		#pass
-
-		output_buffer[x_zero_coord][min_y_coord] = "+"
-		output_buffer[x_zero_coord][max_y_coord] = "+"
-		output_buffer[min_x_coord][y_zero_coord] = "+"
-		output_buffer[max_x_coord][y_zero_coord] = "+"
-
-		if do_plot_x_label:        
-
-			for i,c in enumerate(min_x_str):
-				output_buffer[min_x_coord+i][y_zero_coord-1] = c
-			for i,c in enumerate(max_x_str):
-				output_buffer[max_x_coord+i-len(max_x_str)][y_zero_coord-1] = c
-
-		if do_plot_y_label:
-
-			for i,c in enumerate(max_y_str):
-				output_buffer[x_zero_coord+i][max_y_coord] = c
-			for i,c in enumerate(min_y_str):
-				output_buffer[x_zero_coord+i][min_y_coord] = c
-			
-
-		
-		
-	
-	def plot_data(self, xy_seq, output_buffer, plot_data):
-		if self.plot_slope:
-			xy_seq = list(xy_seq)
-			#sort according to the x coord
-			xy_seq.sort(key = lambda c: c[0])
-			prev_p = xy_seq[0]
-			e_xy_seq = enumerate(xy_seq)
-			e_xy_seq.next()
-			for i,(x,y) in e_xy_seq:
-				draw_symbol = self.dot
-				line_drawn = self.plot_line(prev_p, (x,y), output_buffer, plot_data)
-				prev_p = (x,y)
-				if not line_drawn:
-					if i > 0 and i < len(xy_seq)-1:
-						px,py = xy_seq[i-1]
-						nx,ny = xy_seq[i+1]
-
-						if abs(nx-px) > EPSILON:
-							slope = (1.0/plot_data.ratio)*(ny-py)/(nx-px)
-							draw_symbol = self.get_symbol_by_slope(slope, draw_symbol)
-					if x < plot_data.min_x or x >= plot_data.max_x or y < plot_data.min_y or y >= plot_data.max_y:
-						continue
-					
-					x_coord = self.get_coord(x, plot_data.min_x, plot_data.x_step)
-					y_coord = self.get_coord(y, plot_data.min_y, plot_data.y_step)            
-					if x_coord >= 0 and x_coord < len(output_buffer) and y_coord >= 0 and y_coord < len(output_buffer[0]):
-						if self.draw_axes:
-							if y_coord == self.get_coord(0, plot_data.min_y, plot_data.y_step) and draw_symbol == "-":
-								draw_symbol = "="
-						output_buffer[x_coord][y_coord] = draw_symbol
-		else:
-			for x,y in xy_seq:
-				if x < plot_data.min_x or x >= plot_data.max_x or y < plot_data.min_y or y >= plot_data.max_y:
-					continue
-				x_coord = self.get_coord(x, plot_data.min_x, plot_data.x_step)
-				y_coord = self.get_coord(y, plot_data.min_y, plot_data.y_step)
-				if x_coord >= 0 and x_coord < len(output_buffer) and y_coord > 0 and y_coord < len(output_buffer[0]):
-					output_buffer[x_coord][y_coord] = self.dot
-
-
-	def plot_line(self, start, end, output_buffer, plot_data):
-
-		start_coord = self.get_coord(start[0], plot_data.min_x, plot_data.x_step), self.get_coord(start[1], plot_data.min_y, plot_data.y_step)
-		end_coord = self.get_coord(end[0], plot_data.min_x, plot_data.x_step), self.get_coord(end[1], plot_data.min_y, plot_data.y_step)
-
-		x0,y0 = start_coord
-		x1,y1 = end_coord
-		if (x0,y0) == (x1,y1):
-			return True    
-		
-		clipped_line = clip_line(start, end, (plot_data.min_x, plot_data.min_y), (plot_data.max_x, plot_data.max_y))
-		if clipped_line != None:
-			start,end = clipped_line
-		else:
-			return False
-		start_coord = self.get_coord(start[0], plot_data.min_x, plot_data.x_step), self.get_coord(start[1], plot_data.min_y, plot_data.y_step)
-		end_coord = self.get_coord(end[0], plot_data.min_x, plot_data.x_step), self.get_coord(end[1], plot_data.min_y, plot_data.y_step)
-
-		x0,y0 = start_coord
-		x1,y1 = end_coord
-		if (x0,y0) == (x1,y1):
-			return True
-		x_zero_coord = self.get_coord(0, plot_data.min_x, plot_data.x_step)
-		y_zero_coord = self.get_coord(0, plot_data.min_y, plot_data.y_step)   
-
-		if start[0]-end[0] == 0:
-			draw_symbol = "|"
-		else:
-			slope = (1.0/plot_data.ratio)*(end[1]-start[1])/(end[0]-start[0])
-			draw_symbol = self.get_symbol_by_slope(slope, self.dot)
-		try:
-
-			delta = x1-x0, y1-y0
-			if abs(delta[0])>abs(delta[1]):
-				s = sign(delta[0])
-				slope = float(delta[1])/delta[0]
-				for i in range(0,abs(int(delta[0]))):
-					cur_draw_symbol = draw_symbol
-					x = i*s
-					cur_y = int(y0+slope*x)
-					if self.draw_axes and cur_y == y_zero_coord and draw_symbol == "-":
-						cur_draw_symbol = "="
-					output_buffer[x0+x][cur_y] = cur_draw_symbol
-				
-				
-			else:
-				s = sign(delta[1])
-				slope = float(delta[0])/delta[1]
-				for i in range(0,abs(int(delta[1]))):
-					y = i*s
-					cur_draw_symbol = draw_symbol
-					cur_y = y0+y
-					if self.draw_axes and cur_y == y_zero_coord and draw_symbol == "-":
-						cur_draw_symbol = "="
-					output_buffer[int(x0+slope*y)][cur_y] = cur_draw_symbol
-		except:
-			print start, end
-			print start_coord, end_coord
-			print plot_data
-			raise
-
-		return False            
-		
-		
-	def plot_single(self, seq, min_x = None, max_x = None, min_y = None, max_y = None):
-		return self.plot_double(range(len(seq)),seq, min_x, max_x, min_y, max_y)
-		
-
-
-
-	def plot_double(self, x_seq, y_seq, min_x = None, max_x = None, min_y = None, max_y = None):
-		if min_x == None:
-			min_x = min(x_seq)
-		if max_x == None:
-			max_x = max(x_seq)
-		if min_y == None:
-			min_y = min(y_seq)
-		if max_y == None:
-			max_y = max(y_seq)
-
-		if max_y == min_y:
-			max_y += 1
-
-		x_mod = (max_x-min_x)*self.x_margin
-		y_mod = (max_y-min_y)*self.y_margin
-		min_x-=x_mod
-		max_x+=x_mod
-		min_y-=y_mod
-		max_y+=y_mod
-
-
-		plot_data = self.PlotData(self.x_size, self.y_size, min_x, max_x, min_y, max_y, x_mod, y_mod)
-
-		output_buffer = [[" "]*self.y_size for i in range(self.x_size)]
-
-		if self.will_draw_axes:
-			self.draw_axes(output_buffer, plot_data)
-
-		self.plot_data(zip(x_seq, y_seq), output_buffer, plot_data)
-
-		if self.will_plot_labels:
-			self.plot_labels(output_buffer, plot_data)
-
-		trans_result = transposed(y_reversed(output_buffer))
-
-		result = self.new_line.join(["".join(row) for row in trans_result])
-		return result
-
-	def draw_axes(self, output_buffer, plot_data):
-		
-		
-		draw_x = False
-		draw_y = False
-
-		if plot_data.min_x <= 0 and plot_data.max_x > 0:
-			draw_y = True
-			zero_x = self.get_coord(0, plot_data.min_x, plot_data.x_step)
-			for y in xrange(plot_data.y_size):
-				output_buffer[zero_x][y] = "|"
-				
-		if plot_data.min_y <= 0 and plot_data.max_y > 0:
-			draw_x = True
-			zero_y = self.get_coord(0, plot_data.min_y, plot_data.y_step)    
-			for x in xrange(plot_data.x_size):
-				output_buffer[x][zero_y] = "-"
-
-		if draw_x and draw_y:
-			output_buffer[zero_x][zero_y] = "+"
-		
-		
-	@staticmethod
-	def get_coord(val, min, step):
-		result = int((val - min)/step)
-		return result
-
-def clip_line(line_pt_1, line_pt_2, rect_bottom_left, rect_top_right):
-	ts = [0.0,1.0]
-	if line_pt_1[0] == line_pt_2[0]:
-		return ((line_pt_1[0], max(min(line_pt_1[1], line_pt_2[1]), rect_bottom_left[1])),
-				(line_pt_1[0], min(max(line_pt_1[1], line_pt_2[1]), rect_top_right[1])))
-	if line_pt_1[1] == line_pt_2[1]:
-		return ((max(min(line_pt_1[0], line_pt_2[0]), rect_bottom_left[0]), line_pt_1[1]),
-				(min(max(line_pt_1[0], line_pt_2[0]), rect_top_right[0]), line_pt_1[1]))
-
-	if ((rect_bottom_left[0] <= line_pt_1[0] and line_pt_1[0] < rect_top_right[0]) and
-		(rect_bottom_left[1] <= line_pt_1[1] and line_pt_1[1] < rect_top_right[1]) and
-		(rect_bottom_left[0] <= line_pt_2[0] and line_pt_2[0] < rect_top_right[0]) and
-		(rect_bottom_left[1] <= line_pt_2[1] and line_pt_2[1] < rect_top_right[1])):
-		return line_pt_1, line_pt_2
-
-	ts.append( float(rect_bottom_left[0]-line_pt_1[0])/(line_pt_2[0]-line_pt_1[0]) )
-	ts.append( float(rect_top_right[0]-line_pt_1[0])/(line_pt_2[0]-line_pt_1[0]) )
-	ts.append( float(rect_bottom_left[1]-line_pt_1[1])/(line_pt_2[1]-line_pt_1[1]) )
-	ts.append( float(rect_top_right[1]-line_pt_1[1])/(line_pt_2[1]-line_pt_1[1]) )
-	
-	ts.sort()
-	if ts[2] < 0 or ts[2] >= 1 or ts[3] < 0 or ts[2]>= 1:
-		return None
-	result = [(pt_1 + t*(pt_2-pt_1)) for t in (ts[2],ts[3]) for (pt_1, pt_2) in zip(line_pt_1, line_pt_2)]
-	return (result[0],result[1]), (result[2], result[3])
-	
-
-
-def plot(*args,**flags):
-	limit_flags_names = set(["min_x","min_y","max_x","max_y"])
-	limit_flags = dict([(n,flags[n]) for n in limit_flags_names & set(flags)])
-	settting_flags = dict([(n,flags[n]) for n in set(flags) - limit_flags_names])
-	
-	if len(args) == 1:
-		p = Plotter(**settting_flags)
-		print p.plot_single(args[0],**limit_flags)
-	elif len(args) == 2:
-		p = Plotter(**settting_flags)
-		print p.plot_double(args[0],args[1],**limit_flags)
-	else:
-		raise NotImplementedError("can't draw multiple graphs yet")
-	
-__all__ = ["Plotter","plot"]
-

tests/data/extract-1

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
 # start: Fri, 23 Mar 2012 10:00:30.000000 +0000
 # end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
+1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
+1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
+1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
+1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
+1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
+1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
+1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
+1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
+1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
+1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
+1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
+1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
+1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
+1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
+1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
+1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
+1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
+1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
+1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
+1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
+1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
+1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
+1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
+1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
+1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
+1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
+1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
+1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
+1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
+1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
+1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
+1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
+1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
+1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
+1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
+1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
+1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
+1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
+1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
+1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
+1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
+1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
+1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
+1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
+1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
+1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
+1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
+1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
+1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
+1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
+1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
+1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
+1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
+1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
+1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
+1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
+1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
+1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
+1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
+1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
+1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
+1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
+1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
+1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
+1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
+1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
+1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
+1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
+1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
+1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
+1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
+1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
+1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
+1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
+1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
+1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
+1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
+1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
+1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
+1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
+1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
+1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
+1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
+1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
+1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
+1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
+1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
+1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
+1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
+1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
+1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
+1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
+1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
+1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
+1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
+1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
+1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
+1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
+1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
+1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
+1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
+1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
+1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
+1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
+1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
+1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
+1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
+1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
+1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
+1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
+1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
+1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
+1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
+1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
+1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
+1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
+1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-2

@@ -1,119 +1,119 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
+1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
+1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
+1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
+1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
+1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
+1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
+1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
+1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
+1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
+1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
+1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
+1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
+1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
+1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
+1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
+1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
+1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
+1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
+1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
+1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
+1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
+1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
+1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
+1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
+1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
+1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
+1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
+1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
+1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
+1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
+1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
+1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
+1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
+1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
+1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
+1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
+1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
+1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
+1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
+1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
+1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
+1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
+1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
+1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
+1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
+1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
+1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
+1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
+1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
+1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
+1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
+1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
+1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
+1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
+1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
+1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
+1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
+1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
+1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
+1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
+1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
+1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
+1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
+1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
+1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
+1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
+1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
+1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
+1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
+1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
+1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
+1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
+1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
+1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
+1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
+1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
+1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
+1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
+1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
+1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
+1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
+1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
+1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
+1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
+1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
+1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
+1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
+1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
+1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
+1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
+1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
+1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
+1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
+1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
+1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
+1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
+1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
+1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
+1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
+1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
+1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
+1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
+1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
+1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
+1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
+1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
+1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
+1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
+1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
+1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
+1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
+1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
+1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
+1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
+1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
+1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
+1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-3

@@ -1 +1 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03

tests/data/extract-4

@@ -1,2 +1,2 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03

tests/data/extract-5

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
 # start: Fri, 23 Mar 2012 10:00:30.000000 +0000
 # end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
+2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
+2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
+2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
+2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
+2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
+2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
+2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
+2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
+2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
+2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
+2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
+2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
+2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
+2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
+2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
+2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
+2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
+2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
+2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
+2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
+2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
+2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
+2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
+2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
+2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
+2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
+2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
+2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
+2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
+2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
+2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
+2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
+2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
+2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
+2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
+2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
+2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
+2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
+2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
+2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
+2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
+2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
+2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
+2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
+2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
+2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
+2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
+2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
+2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
+2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
+2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
+2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
+2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
+2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
+2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
+2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
+2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
+2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
+2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
+2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
+2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
+2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
+2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
+2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
+2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
+2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
+2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
+2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
+2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
+2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
+2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
+2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
+2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
+2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
+2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
+2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
+2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
+2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
+2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
+2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
+2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
+2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
+2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
+2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
+2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
+2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
+2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
+2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
+2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
+2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
+2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
+2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
+2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
+2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
+2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
+2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
+2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
+2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
+2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
+2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
+2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
+2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
+2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
+2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
+2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
+2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
+2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
+2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
+2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
+2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
+2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
+2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
+2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
+2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
+2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
+2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
+2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
+2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
+2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-6

@@ -1,120 +1,120 @@
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
+2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
+2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
+2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
+2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
+2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
+2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
+2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
+2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
+2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
+2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
+2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
+2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
+2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
+2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
+2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
+2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
+2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
+2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
+2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
+2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
+2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
+2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
+2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
+2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
+2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
+2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
+2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
+2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
+2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
+2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
+2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
+2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
+2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
+2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
+2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
+2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
+2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
+2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
+2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
+2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
+2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
+2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
+2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
+2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
+2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
+2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
+2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
+2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
+2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
+2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
+2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
+2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
+2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
+2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
+2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
+2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
+2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
+2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
+2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
+2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
+2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
+2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
+2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
+2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
+2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
+2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
+2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
+2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
+2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
+2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
+2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
+2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
+2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
+2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
+2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
+2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
+2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
+2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
+2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
+2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
+2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
+2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
+2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
+2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
+2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
+2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
+2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
+2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
+2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
+2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
+2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
+2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
+2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
+2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
+2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
+2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
+2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
+2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
+2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
+2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
+2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
+2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
+2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
+2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
+2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
+2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
+2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
+2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
+2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
+2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
+2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
+2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
+2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
+2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
+2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
+2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
+2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
+2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
+2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-7

@@ -0,0 +1,124 @@
+# path: /newton/prep
+# layout: float32_8
+# start: 1332496830000000
+# end: 1332496830999000
+1332496830000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
+1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
+1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
+1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
+1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
+1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
+1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
+1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
+1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
+1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
+1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
+1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
+1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
+1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
+1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
+1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
+1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
+1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
+1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
+1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
+1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
+1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
+1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
+1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
+1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
+1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
+1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
+1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
+1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
+1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
+1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
+1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
+1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
+1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
+1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
+1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
+1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
+1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
+1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
+1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
+1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
+1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
+1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
+1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
+1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
+1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
+1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
+1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
+1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
+1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
+1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
+1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
+1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
+1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
+1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
+1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
+1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
+1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
+1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
+1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
+1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
+1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
+1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
+1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
+1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
+1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
+1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
+1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
+1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
+1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
+1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
+1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
+1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
+1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
+1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
+1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
+1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
+1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
+1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
+1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
+1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
+1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
+1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
+1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
+1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
+1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
+1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
+1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
+1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
+1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
+1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
+1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
+1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
+1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
+1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
+1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
+1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
+1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
+1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
+1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
+1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
+1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
+1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
+1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
+1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
+1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
+1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
+1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
+1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
+1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
+1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
+1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
+1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
+1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
+1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
+1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
+1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
+1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
+1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
+1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-8

@@ -0,0 +1,28 @@
+# interval-start 1332496919900000
+1332496919900000 2.523050e+05 2.254020e+05 4.779410e+03 3.638030e+03 8.138070e+03 4.334460e+03 1.083780e+03 3.743730e+03
+1332496919908333 2.551190e+05 2.237870e+05 5.965640e+03 2.076350e+03 9.468790e+03 3.693880e+03 1.247860e+03 3.393680e+03
+1332496919916667 2.616370e+05 2.247980e+05 4.848970e+03 2.315620e+03 9.323300e+03 4.225460e+03 1.805780e+03 2.593050e+03
+1332496919925000 2.606460e+05 2.251300e+05 3.061360e+03 3.951840e+03 7.662910e+03 5.341410e+03 1.986520e+03 2.276780e+03
+1332496919933333 2.559710e+05 2.235030e+05 4.096030e+03 3.296970e+03 7.827080e+03 5.452120e+03 2.492520e+03 2.929450e+03
+1332496919941667 2.579260e+05 2.217080e+05 5.472320e+03 1.555700e+03 8.495760e+03 4.491140e+03 2.379780e+03 3.741710e+03
+1332496919950000 2.610180e+05 2.242350e+05 4.669770e+03 1.876190e+03 8.366680e+03 3.677510e+03 9.021690e+02 3.549040e+03
+1332496919958333 2.569150e+05 2.274650e+05 2.785070e+03 3.751930e+03 7.440320e+03 3.964860e+03 -3.227860e+02 2.460890e+03
+1332496919966667 2.509510e+05 2.262000e+05 3.772710e+03 3.131950e+03 8.159860e+03 4.539860e+03 7.375190e+02 2.126750e+03
+1332496919975000 2.556710e+05 2.223720e+05 5.826200e+03 8.715560e+02 9.120240e+03 4.545110e+03 2.804310e+03 2.721000e+03
+1332496919983333 2.649730e+05 2.214860e+05 5.839130e+03 4.659180e+02 8.628300e+03 3.934870e+03 2.972490e+03 3.773730e+03
+1332496919991667 2.652170e+05 2.233920e+05 3.718770e+03 2.834970e+03 7.209900e+03 3.460260e+03 1.324930e+03 4.075960e+03
+# interval-end 1332496919991668
+# interval-start 1332496920000000
+1332496920000000 2.564370e+05 2.244300e+05 4.011610e+03 3.475340e+03 7.495890e+03 3.388940e+03 2.613970e+02 3.731260e+03
+1332496920008333 2.539630e+05 2.241670e+05 5.621070e+03 1.548010e+03 9.165170e+03 3.522930e+03 1.058930e+03 2.996960e+03
+1332496920016667 2.585080e+05 2.249300e+05 6.011400e+03 8.188660e+02 9.039950e+03 4.482440e+03 2.490390e+03 2.679340e+03
+1332496920025000 2.596270e+05 2.260220e+05 4.474500e+03 2.423020e+03 7.414190e+03 5.071970e+03 2.439380e+03 2.962960e+03
+1332496920033333 2.551870e+05 2.246320e+05 4.738570e+03 3.398040e+03 7.395120e+03 4.726450e+03 1.839030e+03 3.393530e+03
+1332496920041667 2.571020e+05 2.216230e+05 6.144130e+03 1.441090e+03 8.756480e+03 3.495320e+03 1.869940e+03 3.752530e+03
+1332496920050000 2.636530e+05 2.217700e+05 6.221770e+03 7.389620e+02 9.547600e+03 2.666820e+03 1.462660e+03 3.332570e+03
+1332496920058333 2.636130e+05 2.252560e+05 4.477120e+03 2.437450e+03 8.510210e+03 3.855630e+03 9.594420e+02 2.387180e+03
+1332496920066667 2.553500e+05 2.262640e+05 4.283720e+03 3.923940e+03 7.912470e+03 5.466520e+03 1.284990e+03 2.093720e+03
+1332496920075000 2.527270e+05 2.246090e+05 5.851930e+03 2.491980e+03 8.540630e+03 5.623050e+03 2.339780e+03 3.007140e+03
+1332496920083333 2.584750e+05 2.235780e+05 5.924870e+03 1.394480e+03 8.779620e+03 4.544180e+03 2.132030e+03 3.849760e+03
+1332496920091667 2.615630e+05 2.246090e+05 4.336140e+03 2.455750e+03 8.055380e+03 3.469110e+03 6.278730e+02 3.664200e+03
+# interval-end 1332496920100000

tests/data/prep-20120323T1000

@@ -1,3 +1,4 @@
+# comments are cool?
 2.66568e+05  2.24029e+05  5.16140e+03  2.52517e+03  8.35084e+03  3.72470e+03  1.35534e+03  2.03900e+03  
 2.57914e+05  2.27183e+05  4.30368e+03  4.13080e+03  7.25535e+03  4.89047e+03  1.63859e+03  1.93496e+03  
 2.51717e+05  2.26047e+05  5.99445e+03  3.49363e+03  8.07250e+03  5.08267e+03  2.26917e+03  2.86231e+03  

tests/data/prep-20120323T1002-first19lines

@@ -0,0 +1,19 @@
+2.56437e+05  2.24430e+05  4.01161e+03  3.47534e+03  7.49589e+03  3.38894e+03  2.61397e+02  3.73126e+03  
+2.53963e+05  2.24167e+05  5.62107e+03  1.54801e+03  9.16517e+03  3.52293e+03  1.05893e+03  2.99696e+03  
+2.58508e+05  2.24930e+05  6.01140e+03  8.18866e+02  9.03995e+03  4.48244e+03  2.49039e+03  2.67934e+03  
+2.59627e+05  2.26022e+05  4.47450e+03  2.42302e+03  7.41419e+03  5.07197e+03  2.43938e+03  2.96296e+03  
+2.55187e+05  2.24632e+05  4.73857e+03  3.39804e+03  7.39512e+03  4.72645e+03  1.83903e+03  3.39353e+03  
+2.57102e+05  2.21623e+05  6.14413e+03  1.44109e+03  8.75648e+03  3.49532e+03  1.86994e+03  3.75253e+03  
+2.63653e+05  2.21770e+05  6.22177e+03  7.38962e+02  9.54760e+03  2.66682e+03  1.46266e+03  3.33257e+03  
+2.63613e+05  2.25256e+05  4.47712e+03  2.43745e+03  8.51021e+03  3.85563e+03  9.59442e+02  2.38718e+03  
+2.55350e+05  2.26264e+05  4.28372e+03  3.92394e+03  7.91247e+03  5.46652e+03  1.28499e+03  2.09372e+03  
+2.52727e+05  2.24609e+05  5.85193e+03  2.49198e+03  8.54063e+03  5.62305e+03  2.33978e+03  3.00714e+03  
+2.58475e+05  2.23578e+05  5.92487e+03  1.39448e+03  8.77962e+03  4.54418e+03  2.13203e+03  3.84976e+03  
+2.61563e+05  2.24609e+05  4.33614e+03  2.45575e+03  8.05538e+03  3.46911e+03  6.27873e+02  3.66420e+03  
+2.56401e+05  2.24441e+05  4.18715e+03  3.45717e+03  7.90669e+03  3.53355e+03  -5.84482e+00  2.96687e+03  
+2.54745e+05  2.22644e+05  6.02005e+03  1.94721e+03  9.28939e+03  3.80020e+03  1.34820e+03  2.37785e+03  
+2.60723e+05  2.22660e+05  6.69719e+03  1.03048e+03  9.26124e+03  4.34917e+03  2.84530e+03  2.73619e+03  
+2.63089e+05  2.25711e+05  4.77887e+03  2.60417e+03  7.39660e+03  4.59811e+03  2.17472e+03  3.40729e+03  
+2.55843e+05  2.27128e+05  4.02413e+03  4.39323e+03  6.79336e+03  4.62535e+03  7.52009e+02  3.44647e+03  
+2.51904e+05  2.24868e+05  5.82289e+03  3.02127e+03  8.46160e+03  3.80298e+03  8.07212e+02  3.53468e+03  
+2.57670e+05  2.22974e+05  6.73436e+03  1.60956e+03  9.92960e+03  2.98028e+03  1.44168e+03  3.05351e+03  

tests/data/prep-20120323T1004-badtimes

@@ -0,0 +1,11 @@
+1332497040000000 2.56439e+05  2.24775e+05  2.92897e+03  4.66646e+03  7.58491e+03  3.57351e+03  -4.34171e+02  2.98819e+03  
+1332497040010000 2.51903e+05  2.23202e+05  4.23696e+03  3.49363e+03  8.53493e+03  4.29416e+03  8.49573e+02  2.38189e+03  
+1332497040020000 2.57625e+05  2.20247e+05  5.47017e+03  1.35872e+03  9.18903e+03  4.56136e+03  2.65599e+03  2.60912e+03  
+1332497040030000 2.63375e+05  2.20706e+05  4.51842e+03  1.80758e+03  8.17208e+03  4.17463e+03  2.57884e+03  3.32848e+03  
+1332497040040000 2.59221e+05  2.22346e+05  2.98879e+03  3.66264e+03  6.87274e+03  3.94223e+03  1.25928e+03  3.51786e+03  
+1332497040050000 2.51918e+05  2.22281e+05  4.22677e+03  2.84764e+03  7.78323e+03  3.81659e+03  8.04944e+02  3.46314e+03  
+1332497040050000 2.54478e+05  2.21701e+05  5.61366e+03  1.02262e+03  9.26581e+03  3.50152e+03  1.29331e+03  3.07271e+03  
+1332497040060000 2.59568e+05  2.22945e+05  4.97190e+03  1.28250e+03  8.62081e+03  4.06316e+03  1.85717e+03  2.61990e+03  
+1332497040070000 2.57269e+05  2.23697e+05  3.60527e+03  3.05749e+03  7.22363e+03  4.90330e+03  1.93736e+03  2.35357e+03  
+1332497040080000 2.52274e+05  2.21438e+05  5.01228e+03  2.86309e+03  7.87115e+03  4.80448e+03  2.18291e+03  2.93397e+03  
+1332497040090000 2.56468e+05  2.19205e+05  6.29804e+03  8.09467e+02  9.12895e+03  3.52055e+03  2.16980e+03  3.88739e+03  

tests/runtests.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python
+
+import nose
+import os
+import sys
+import glob
+from collections import OrderedDict
+
+# Change into parent dir
+os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/..")
+
+class JimOrderPlugin(nose.plugins.Plugin):
+    """When searching for tests and encountering a directory that
+    contains a 'test.order' file, run tests listed in that file, in the
+    order that they're listed.  Globs are OK in that file and duplicates
+    are removed."""
+    name = 'jimorder'
+    score = 10000
+
+    def prepareTestLoader(self, loader):
+        def wrap(func):
+            def wrapper(name, *args, **kwargs):
+                addr = nose.selector.TestAddress(
+                    name, workingDir=loader.workingDir)
+                try:
+                    order = os.path.join(addr.filename, "test.order")
+                except Exception:
+                    order = None
+                if order and os.path.exists(order):
+                    files = []
+                    for line in open(order):
+                        line = line.split('#')[0].strip()
+                        if not line:
+                            continue
+                        fn = os.path.join(addr.filename, line.strip())
+                        files.extend(sorted(glob.glob(fn)) or [fn])
+                    files = list(OrderedDict.fromkeys(files))
+                    tests = [ wrapper(fn, *args, **kwargs) for fn in files ]
+                    return loader.suiteClass(tests)
+                return func(name, *args, **kwargs)
+            return wrapper
+        loader.loadTestsFromName = wrap(loader.loadTestsFromName)
+        return loader
+
+# Use setup.cfg for most of the test configuration.  Adding
+# --with-jimorder here means that a normal "nosetests" run will
+# still work, it just won't support test.order.
+nose.main(addplugins = [ JimOrderPlugin() ],
+          argv = sys.argv + ["--with-jimorder"])

tests/test.order

@@ -0,0 +1,17 @@
+test_printf.py
+test_threadsafety.py
+test_lrucache.py
+test_mustclose.py
+
+test_serializer.py
+
+test_timestamper.py
+test_rbtree.py
+test_interval.py
+
+test_bulkdata.py
+test_nilmdb.py
+test_client.py
+test_cmdline.py
+
+test_*.py

tests/test_bulkdata.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+
+import nilmdb
+from nilmdb.utils.printf import *
+from nose.tools import *
+from nose.tools import assert_raises
+import itertools
+
+from testutil.helpers import *
+
+testdb = "tests/bulkdata-testdb"
+
+import nilmdb.server.bulkdata
+from nilmdb.server.bulkdata import BulkData
+
+class TestBulkData(object):
+
+    def test_bulkdata(self):
+        for (size, files, db) in [ ( 0, 0, testdb ),
+                                       ( 25, 1000, testdb ),
+                                       ( 1000, 3, testdb.decode("utf-8") ) ]:
+            recursive_unlink(db)
+            os.mkdir(db)
+            self.do_basic(db, size, files)
+
+    def do_basic(self, db, size, files):
+        """Do the basic test with variable file_size and files_per_dir"""
+        if not size or not files:
+            data = BulkData(db)
+        else:
+            data = BulkData(db, file_size = size, files_per_dir = files)
+
+        # Try opening it again (should result in locking error)
+        with assert_raises(IOError) as e:
+            data2 = BulkData(db)
+        in_("already locked by another process", str(e.exception))
+
+        # create empty
+        with assert_raises(ValueError):
+            data.create("/foo", "uint16_8")
+        with assert_raises(ValueError):
+            data.create("foo/bar", "uint16_8")
+        data.create("/foo/bar", "uint16_8")
+        data.create(u"/foo/baz/quux", "float64_16")
+        with assert_raises(ValueError):
+            data.create("/foo/bar/baz", "uint16_8")
+        with assert_raises(ValueError):
+            data.create("/foo/baz", "float64_16")
+
+        # get node -- see if caching works
+        nodes = []
+        for i in range(5000):
+            nodes.append(data.getnode("/foo/bar"))
+            nodes.append(data.getnode("/foo/baz/quux"))
+        del nodes
+
+        def get_node_slice(key):
+            if isinstance(key, slice):
+                return [ node.get_data(x, x+1) for x in
+                         xrange(*key.indices(node.nrows)) ]
+            return node.get_data(key, key+1)
+
+        # Test node
+        node = data.getnode("/foo/bar")
+        with assert_raises(IndexError):
+            x = get_node_slice(0)
+        with assert_raises(IndexError):
+            x = node[0] # timestamp
+        raw = []
+        for i in range(1000):
+            raw.append("%d 1 2 3 4 5 6 7 8\n" % (10000 + i))
+        node.append_string("".join(raw[0:1]), 0, 50000)
+        node.append_string("".join(raw[1:100]), 0, 50000)
+        node.append_string("".join(raw[100:]), 0, 50000)
+
+        misc_slices = [ 0, 100, slice(None), slice(0), slice(10),
+                        slice(5,10), slice(3,None), slice(3,-3),
+                        slice(20,10), slice(200,100,-1), slice(None,0,-1),
+                        slice(100,500,5) ]
+
+        # Extract slices
+        for s in misc_slices:
+            eq_(get_node_slice(s), raw[s])
+
+        # Extract misc slices while appending, to make sure the
+        # data isn't being added in the middle of the file
+        for s in [2, slice(1,5), 2, slice(1,5)]:
+            node.append_string("0 0 0 0 0 0 0 0 0\n", 0, 50000)
+            raw.append("0 0 0 0 0 0 0 0 0\n")
+            eq_(get_node_slice(s), raw[s])
+
+        # Get some coverage of remove; remove is more fully tested
+        # in cmdline
+        with assert_raises(IndexError):
+            node.remove(9999,9998)
+
+        # close, reopen
+        # reopen
+        data.close()
+        if not size or not files:
+            data = BulkData(db)
+        else:
+            data = BulkData(db, file_size = size, files_per_dir = files)
+        node = data.getnode("/foo/bar")
+
+        # Extract slices
+        for s in misc_slices:
+            eq_(get_node_slice(s), raw[s])
+
+        # destroy
+        with assert_raises(ValueError):
+            data.destroy("/foo")
+        with assert_raises(ValueError):
+            data.destroy("/foo/baz")
+        with assert_raises(ValueError):
+            data.destroy("/foo/qwerty")
+        data.destroy("/foo/baz/quux")
+        data.destroy("/foo/bar")
+
+        # close
+        data.close()

tests/test_client.py

@@ -1,11 +1,14 @@
 # -*- coding: utf-8 -*-
 
-import nilmdb
+import nilmdb.server
+import nilmdb.client
+
 from nilmdb.utils.printf import *
+from nilmdb.utils import timestamper
 from nilmdb.client import ClientError, ServerError
+from nilmdb.utils import datetime_tz
 
-import datetime_tz
-
+from nose.plugins.skip import SkipTest
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
@@ -17,10 +20,14 @@ import cStringIO
 import simplejson as json
 import unittest
 import warnings
+import resource
+import time
+import re
 
-from test_helpers import *
+from testutil.helpers import *
 
 testdb = "tests/client-testdb"
+testurl = "http://localhost:32180/"
 
 def setup_module():
     global test_server, test_db
@@ -28,11 +35,11 @@ def setup_module():
     recursive_unlink(testdb)
 
     # Start web app on a custom port
-    test_db = nilmdb.NilmDB(testdb, sync = False)
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(testdb)
+    test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
+                                       port = 32180, stoppable = False,
                                        fast_shutdown = True,
-                                force_traceback = False)
+                                       force_traceback = True)
     test_server.start(blocking = False)
 
 def teardown_module():
@@ -43,61 +50,90 @@ def teardown_module():
 
 class TestClient(object):
 
-    def test_client_1_basic(self):
+    def test_client_01_basic(self):
         # Test a fake host
-        client = nilmdb.Client(url = "http://localhost:1/")
-        with assert_raises(nilmdb.client.ServerError):
-            client.version()
-
-        # Trigger same error with a PUT request
-        client = nilmdb.Client(url = "http://localhost:1/")
+        client = nilmdb.client.Client(url = "http://localhost:1/")
         with assert_raises(nilmdb.client.ServerError):
             client.version()
+        client.close()
 
         # Then a fake URL on a real host
-        client = nilmdb.Client(url = "http://localhost:12380/fake/")
+        client = nilmdb.client.Client(url = "http://localhost:32180/fake/")
         with assert_raises(nilmdb.client.ClientError):
             client.version()
+        client.close()
 
         # Now a real URL with no http:// prefix
-        client = nilmdb.Client(url = "localhost:12380")
+        client = nilmdb.client.Client(url = "localhost:32180")
         version = client.version()
+        client.close()
 
         # Now use the real URL
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
         version = client.version()
-        eq_(distutils.version.StrictVersion(version),
-            distutils.version.StrictVersion(test_server.version))
+        eq_(distutils.version.LooseVersion(version),
+            distutils.version.LooseVersion(test_server.version))
 
-    def test_client_2_nilmdb(self):
+        # Bad URLs should give 404, not 500
+        with assert_raises(ClientError):
+            client.http.get("/stream/create")
+        client.close()
+
+    def test_client_02_createlist(self):
         # Basic stream tests, like those in test_nilmdb:test_stream
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
 
         # Database starts empty
         eq_(client.stream_list(), [])
 
         # Bad path
         with assert_raises(ClientError):
-            client.stream_create("foo/bar/baz", "PrepData")
+            client.stream_create("foo/bar/baz", "float32_8")
         with assert_raises(ClientError):
-            client.stream_create("/foo", "PrepData")
+            client.stream_create("/foo", "float32_8")
         # Bad layout type
         with assert_raises(ClientError):
             client.stream_create("/newton/prep", "NoSuchLayout")
 
+        # Bad method types
+        with assert_raises(ClientError):
+            client.http.put("/stream/list","")
+        # Try a bunch of times to make sure the request body is getting consumed
+        for x in range(10):
+            with assert_raises(ClientError):
+                client.http.post("/stream/list")
+        client = nilmdb.client.Client(url = testurl)
+
         # Create three streams
-        client.stream_create("/newton/prep", "PrepData")
-        client.stream_create("/newton/raw", "RawData")
-        client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
+        client.stream_create("/newton/prep", "float32_8")
+        client.stream_create("/newton/raw", "uint16_6")
+        client.stream_create("/newton/zzz/rawnotch", "uint16_9")
 
         # Verify we got 3 streams
-        eq_(client.stream_list(), [ ["/newton/prep", "PrepData"],
-                                    ["/newton/raw", "RawData"],
-                                    ["/newton/zzz/rawnotch", "RawNotchedData"]
+        eq_(client.stream_list(), [ ["/newton/prep", "float32_8"],
+                                    ["/newton/raw", "uint16_6"],
+                                    ["/newton/zzz/rawnotch", "uint16_9"]
                                     ])
         # Match just one type or one path
-        eq_(client.stream_list(layout="RawData"), [ ["/newton/raw", "RawData"] ])
-        eq_(client.stream_list(path="/newton/raw"), [ ["/newton/raw", "RawData"] ])
+        eq_(client.stream_list(layout="uint16_6"),
+            [ ["/newton/raw", "uint16_6"] ])
+        eq_(client.stream_list(path="/newton/raw"),
+            [ ["/newton/raw", "uint16_6"] ])
+
+        # Try messing with resource limits to trigger errors and get
+        # more coverage.  Here, make it so we can only create files 1
+        # byte in size, which will trigger an IOError in the server when
+        # we create a table.
+        limit = resource.getrlimit(resource.RLIMIT_FSIZE)
+        resource.setrlimit(resource.RLIMIT_FSIZE, (1, limit[1]))
+        with assert_raises(ServerError) as e:
+            client.stream_create("/newton/hello", "uint16_6")
+        resource.setrlimit(resource.RLIMIT_FSIZE, limit)
+
+        client.close()
+
+    def test_client_03_metadata(self):
+        client = nilmdb.client.Client(url = testurl)
 
         # Set / get metadata
         eq_(client.stream_get_metadata("/newton/prep"), {})
@@ -112,9 +148,10 @@ class TestClient(object):
         client.stream_update_metadata("/newton/raw", meta3)
         eq_(client.stream_get_metadata("/newton/prep"), meta1)
         eq_(client.stream_get_metadata("/newton/raw"), meta1)
-        eq_(client.stream_get_metadata("/newton/raw", [ "description" ] ), meta2)
-        eq_(client.stream_get_metadata("/newton/raw", [ "description",
-                                                        "v_scale" ] ), meta1)
+        eq_(client.stream_get_metadata("/newton/raw",
+                                       [ "description" ] ), meta2)
+        eq_(client.stream_get_metadata("/newton/raw",
+                                       [ "description", "v_scale" ] ), meta1)
 
         # missing key
         eq_(client.stream_get_metadata("/newton/raw", "descr"),
@@ -128,129 +165,157 @@ class TestClient(object):
         with assert_raises(ClientError):
             client.stream_update_metadata("/newton/prep", [1,2,3])
 
-    def test_client_3_insert(self):
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        # test wrong types (dict of non-strings)
+        # numbers are OK; they'll get converted to strings
+        client.stream_set_metadata("/newton/prep", { "hello": 1234 })
+        # anything else is not
+        with assert_raises(ClientError):
+            client.stream_set_metadata("/newton/prep", { "world": { 1: 2 } })
+        with assert_raises(ClientError):
+            client.stream_set_metadata("/newton/prep", { "world": [ 1, 2 ] })
+
+        client.close()
+
+    def test_client_04_insert(self):
+        client = nilmdb.client.Client(url = testurl)
+
+        # Limit _max_data to 1 MB, since our test file is 1.5 MB
+        old_max_data = nilmdb.client.client.StreamInserter._max_data
+        nilmdb.client.client.StreamInserter._max_data = 1 * 1024 * 1024
 
         datetime_tz.localtz_set("America/New_York")
 
         testfile = "tests/data/prep-20120323T1000"
-        start = datetime_tz.datetime_tz.smartparse("20120323T1000")
-        start = start.totimestamp()
+        start = nilmdb.utils.time.parse_time("20120323T1000")
         rate = 120
 
         # First try a nonexistent path
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/no-such-path", data)
         in_("404 Not Found", str(e.exception))
 
         # Now try reversed timestamps
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         data = reversed(list(data))
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
         in_("400 Bad Request", str(e.exception))
-        in_("timestamp is not monotonically increasing", str(e.exception))
+        in2_("timestamp is not monotonically increasing",
+             "start must precede end", str(e.exception))
 
         # Now try empty data (no server request made)
         empty = cStringIO.StringIO("")
-        data = nilmdb.timestamper.TimestamperRate(empty, start, 120)
+        data = timestamper.TimestamperRate(empty, start, 120)
         result = client.stream_insert("/newton/prep", data)
         eq_(result, None)
 
-        # Try forcing a server request with empty data
+        # It's OK to insert an empty interval
+        client.http.put("stream/insert", "", { "path": "/newton/prep",
+                                               "start": 1, "end": 2 })
+        eq_(list(client.stream_intervals("/newton/prep")), [[1, 2]])
+        client.stream_remove("/newton/prep")
+        eq_(list(client.stream_intervals("/newton/prep")), [])
+
+        # Timestamps can be negative too
+        client.http.put("stream/insert", "", { "path": "/newton/prep",
+                                               "start": -2, "end": -1 })
+        eq_(list(client.stream_intervals("/newton/prep")), [[-2, -1]])
+        client.stream_remove("/newton/prep")
+        eq_(list(client.stream_intervals("/newton/prep")), [])
+
+        # Intervals that end at zero shouldn't be any different
+        client.http.put("stream/insert", "", { "path": "/newton/prep",
+                                               "start": -1, "end": 0 })
+        eq_(list(client.stream_intervals("/newton/prep")), [[-1, 0]])
+        client.stream_remove("/newton/prep")
+        eq_(list(client.stream_intervals("/newton/prep")), [])
+
+        # Try forcing a server request with equal start and end
         with assert_raises(ClientError) as e:
             client.http.put("stream/insert", "", { "path": "/newton/prep",
                                                    "start": 0, "end": 0 })
         in_("400 Bad Request", str(e.exception))
-        in_("no data provided", str(e.exception))
+        in_("start must precede end", str(e.exception))
 
         # Specify start/end (starts too late)
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
-                                          start + 5, start + 120)
+                                          start + 5000000, start + 120000000)
         in_("400 Bad Request", str(e.exception))
-        in_("Data timestamp 1332511200.0 < start time 1332511205.0",
+        in_("Data timestamp 1332511200000000 < start time 1332511205000000",
             str(e.exception))
 
         # Specify start/end (ends too early)
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
-                                          start, start + 1)
+                                          start, start + 1000000)
         in_("400 Bad Request", str(e.exception))
         # Client chunks the input, so the exact timestamp here might change
         # if the chunk positions change.
-        in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
-            str(e.exception))
+        assert(re.search("Data timestamp 13325[0-9]+ "
+                         ">= end time 1332511201000000", str(e.exception))
+               is not None)
 
         # Now do the real load
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         result = client.stream_insert("/newton/prep", data,
-                                      start, start + 119.999777)
-        eq_(result, "ok")
+                                      start, start + 119999777)
 
         # Verify the intervals.  Should be just one, even if the data
         # was inserted in chunks, due to nilmdb interval concatenation.
         intervals = list(client.stream_intervals("/newton/prep"))
-        eq_(intervals, [[start, start + 119.999777]])
+        eq_(intervals, [[start, start + 119999777]])
 
         # Try some overlapping data -- just insert it again
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
         in_("400 Bad Request", str(e.exception))
-        in_("OverlapError", str(e.exception))
+        in_("verlap", str(e.exception))
 
-    def test_client_4_extract(self):
-        # Misc tests for extract.  Most of them are in test_cmdline.
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        nilmdb.client.client.StreamInserter._max_data = old_max_data
+        client.close()
 
-        for x in client.stream_extract("/newton/prep", 123, 123):
-            raise Exception("shouldn't be any data for this request")
+    def test_client_05_extractremove(self):
+        # Misc tests for extract and remove.  Most of them are in test_cmdline.
+        client = nilmdb.client.Client(url = testurl)
 
-    def test_client_5_generators(self):
+        for x in client.stream_extract("/newton/prep",
+                                       999123000000, 999124000000):
+            raise AssertionError("shouldn't be any data for this request")
+
+        with assert_raises(ClientError) as e:
+            client.stream_remove("/newton/prep", 123000000, 120000000)
+
+        # Test count
+        eq_(client.stream_count("/newton/prep"), 14400)
+
+        client.close()
+
+    def test_client_06_generators(self):
         # A lot of the client functionality is already tested by test_cmdline,
         # but this gets a bit more coverage that cmdline misses.
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
 
         # Trigger a client error in generator
-        start = datetime_tz.datetime_tz.smartparse("20120323T2000")
-        end = datetime_tz.datetime_tz.smartparse("20120323T1000")
+        start = nilmdb.utils.time.parse_time("20120323T2000")
+        end = nilmdb.utils.time.parse_time("20120323T1000")
         for function in [ client.stream_intervals, client.stream_extract ]:
             with assert_raises(ClientError) as e:
-                function("/newton/prep",
-                         start.totimestamp(),
-                         end.totimestamp()).next()
+                function("/newton/prep", start, end).next()
             in_("400 Bad Request", str(e.exception))
-            in_("end before start", str(e.exception))
+            in_("start must precede end", str(e.exception))
 
         # Trigger a curl error in generator
         with assert_raises(ServerError) as e:
-            client.http.get_gen("http://nosuchurl/").next()
+            client.http.get_gen("http://nosuchurl.example.com./").next()
 
         # Trigger a curl error in generator
         with assert_raises(ServerError) as e:
-            client.http.get_gen("http://nosuchurl/").next()
-
-        # Check non-json version of string output
-        eq_(json.loads(client.http.get("/stream/list",retjson=False)),
-            client.http.get("/stream/list",retjson=True))
-
-        # Check non-json version of generator output
-        for (a, b) in itertools.izip(
-            client.http.get_gen("/stream/list",retjson=False),
-            client.http.get_gen("/stream/list",retjson=True)):
-            eq_(json.loads(a), b)
-
-        # Check PUT with generator out
-        with assert_raises(ClientError) as e:
-            client.http.put_gen("stream/insert", "",
-                                { "path": "/newton/prep",
-                                  "start": 0, "end": 0 }).next()
-        in_("400 Bad Request", str(e.exception))
-        in_("no data provided", str(e.exception))
+            client.http.get_gen("http://nosuchurl.example.com./").next()
 
         # Check 404 for missing streams
         for function in [ client.stream_intervals, client.stream_extract ]:
@@ -259,35 +324,58 @@ class TestClient(object):
             in_("404 Not Found", str(e.exception))
             in_("No such stream", str(e.exception))
 
-    def test_client_6_chunked(self):
+        client.close()
+
+    def test_client_07_headers(self):
         # Make sure that /stream/intervals and /stream/extract
-        # properly return streaming, chunked response.  Pokes around
-        # in client.http internals a bit to look at the response
-        # headers.
+        # properly return streaming, chunked, text/plain response.
+        # Pokes around in client.http internals a bit to look at the
+        # response headers.
 
-        client = nilmdb.Client(url = "http://localhost:12380/")
+        client = nilmdb.client.Client(url = testurl)
+        http = client.http
 
-        # Use a warning rather than returning a test failure, so that we can
-        # still disable chunked responses for debugging.
-        x = client.http.get("stream/intervals", { "path": "/newton/prep" },
-                            retjson=False)
-        lines_(x, 1)
-        if "transfer-encoding: chunked" not in client.http._headers.lower():
+        # Use a warning rather than returning a test failure for the
+        # transfer-encoding, so that we can still disable chunked
+        # responses for debugging.
+
+        def headers():
+            h = ""
+            for (k, v) in http._last_response.headers.items():
+                h += k + ": " + v + "\n"
+            return h.lower()
+
+        # Intervals
+        x = http.get("stream/intervals", { "path": "/newton/prep" })
+        if "transfer-encoding: chunked" not in headers():
             warnings.warn("Non-chunked HTTP response for /stream/intervals")
+        if "content-type: application/x-json-stream" not in headers():
+            raise AssertionError("/stream/intervals content type "
+                                 "is not application/x-json-stream:\n" +
+                                 headers())
 
-        x = client.http.get("stream/extract",
+        # Extract
+        x = http.get("stream/extract",
                             { "path": "/newton/prep",
                               "start": "123",
-                              "end": "123" }, retjson=False)
-        if "transfer-encoding: chunked" not in client.http._headers.lower():
+                              "end": "124" })
+        if "transfer-encoding: chunked" not in headers():
             warnings.warn("Non-chunked HTTP response for /stream/extract")
+        if "content-type: text/plain;charset=utf-8" not in headers():
+            raise AssertionError("/stream/extract is not text/plain:\n" +
+                                 headers())
 
-    def test_client_7_unicode(self):
+        client.close()
+
+    def test_client_08_unicode(self):
+        # Try both with and without posting JSON
+        for post_json in (False, True):
             # Basic Unicode tests
-        client = nilmdb.Client(url = "http://localhost:12380/")
+            client = nilmdb.client.Client(url = testurl, post_json = post_json)
 
             # Delete streams that exist
             for stream in client.stream_list():
+                client.stream_remove(stream[0])
                 client.stream_destroy(stream[0])
 
             # Database is empty
@@ -318,3 +406,269 @@ class TestClient(object):
             eq_(client.stream_get_metadata(raw[0]), meta1)
             eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
             eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
+
+            client.close()
+
+    def test_client_09_closing(self):
+        # Make sure we actually close sockets correctly.  New
+        # connections will block for a while if they're not, since the
+        # server will stop accepting new connections.
+        for test in [1, 2]:
+            start = time.time()
+            for i in range(50):
+                if time.time() - start > 15:
+                    raise AssertionError("Connections seem to be blocking... "
+                                         "probably not closing properly.")
+                if test == 1:
+                    # explicit close
+                    client = nilmdb.client.Client(url = testurl)
+                    with assert_raises(ClientError) as e:
+                        client.stream_remove("/newton/prep", 123, 120)
+                    client.close() # remove this to see the failure
+                elif test == 2:
+                    # use the context manager
+                    with nilmdb.client.Client(url = testurl) as c:
+                        with assert_raises(ClientError) as e:
+                            c.stream_remove("/newton/prep", 123, 120)
+
+    def test_client_10_context(self):
+        # Test using the client's stream insertion context manager to
+        # insert data.
+        client = nilmdb.client.Client(testurl)
+
+        client.stream_create("/context/test", "uint16_1")
+        with client.stream_insert_context("/context/test") as ctx:
+            # override _max_data to trigger frequent server updates
+            ctx._max_data = 15
+
+            ctx.insert("100 1\n")
+
+            ctx.insert("101 ")
+            ctx.insert("1\n102 1")
+            ctx.insert("")
+            ctx.insert("\n103 1\n")
+
+            ctx.insert("104 1\n")
+            ctx.insert("# hello\n")
+            ctx.insert("   # hello\n")
+            ctx.insert("  105 1\n")
+            ctx.finalize()
+
+            ctx.insert("107 1\n")
+            ctx.update_end(108)
+            ctx.finalize()
+            ctx.update_start(109)
+            ctx.insert("110 1\n")
+            ctx.insert("111 1\n")
+            ctx.send()
+            ctx.insert("112 1\n")
+            ctx.insert("113 1\n")
+            ctx.insert("114 1\n")
+            ctx.update_end(116)
+            ctx.insert("115 1\n")
+            ctx.update_end(117)
+            ctx.insert("116 1\n")
+            ctx.update_end(118)
+            ctx.insert("117 1" +
+                       " # this is super long" * 100 +
+                       "\n")
+            ctx.finalize()
+            ctx.insert("# this is super long" * 100)
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test", 100, 200) as ctx:
+                ctx.insert("118 1\n")
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test", 200, 300) as ctx:
+                ctx.insert("118 1\n")
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test") as ctx:
+                ctx.insert("bogus data\n")
+
+        with client.stream_insert_context("/context/test", 200, 300) as ctx:
+            # make sure our override wasn't permanent
+            ne_(ctx._max_data, 15)
+            ctx.insert("225 1\n")
+            ctx.finalize()
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test", 300, 400) as ctx:
+                ctx.insert("301 1\n")
+                ctx.insert("302 2\n")
+                ctx.insert("303 3\n")
+                ctx.insert("304 4\n")
+                ctx.insert("304 4\n") # non-monotonic after a few lines
+                ctx.finalize()
+
+        eq_(list(client.stream_intervals("/context/test")),
+            [ [ 100, 106 ],
+              [ 107, 108 ],
+              [ 109, 118 ],
+              [ 200, 300 ] ])
+
+        # destroy stream (try without removing data first)
+        with assert_raises(ClientError):
+            client.stream_destroy("/context/test")
+        client.stream_remove("/context/test")
+        client.stream_destroy("/context/test")
+        client.close()
+
+    def test_client_11_emptyintervals(self):
+        # Empty intervals are ok!  If recording detection events
+        # by inserting rows into the database, we want to be able to
+        # have an interval where no events occurred.  Test them here.
+        client = nilmdb.client.Client(testurl)
+        client.stream_create("/empty/test", "uint16_1")
+
+        def info():
+            result = []
+            for interval in list(client.stream_intervals("/empty/test")):
+                result.append((client.stream_count("/empty/test", *interval),
+                               interval))
+            return result
+
+        eq_(info(), [])
+
+        # Insert a region with just a few points
+        with client.stream_insert_context("/empty/test") as ctx:
+            ctx.update_start(100)
+            ctx.insert("140 1\n")
+            ctx.insert("150 1\n")
+            ctx.insert("160 1\n")
+            ctx.update_end(200)
+            ctx.finalize()
+
+        eq_(info(), [(3, [100, 200])])
+
+        # Delete chunk, which will leave one data point and two intervals
+        client.stream_remove("/empty/test", 145, 175)
+        eq_(info(), [(1, [100, 145]),
+                     (0, [175, 200])])
+
+        # Try also creating a completely empty interval from scratch,
+        # in a few different ways.
+        client.stream_insert("/empty/test", "", 300, 350)
+        client.stream_insert("/empty/test", [], 400, 450)
+        with client.stream_insert_context("/empty/test", 500, 550):
+            pass
+
+        # If enough timestamps aren't provided, empty streams won't be created.
+        client.stream_insert("/empty/test", [])
+        with client.stream_insert_context("/empty/test"):
+            pass
+        client.stream_insert("/empty/test", [], start = 600)
+        with client.stream_insert_context("/empty/test", start = 700):
+            pass
+        client.stream_insert("/empty/test", [], end = 850)
+        with client.stream_insert_context("/empty/test", end = 950):
+            pass
+
+        # Try various things that might cause problems
+        with client.stream_insert_context("/empty/test", 1000, 1050):
+            ctx.finalize() # inserts [1000, 1050]
+            ctx.finalize() # nothing
+            ctx.finalize() # nothing
+            ctx.insert("1100 1\n")
+            ctx.finalize() # inserts [1100, 1101]
+            ctx.update_start(1199)
+            ctx.insert("1200 1\n")
+            ctx.update_end(1250)
+            ctx.finalize() # inserts [1199, 1250]
+            ctx.update_start(1299)
+            ctx.finalize() # nothing
+            ctx.update_end(1350)
+            ctx.finalize() # nothing
+            ctx.update_start(1400)
+            ctx.insert("# nothing!\n")
+            ctx.update_end(1450)
+            ctx.finalize()
+            ctx.update_start(1500)
+            ctx.insert("# nothing!")
+            ctx.update_end(1550)
+            ctx.finalize()
+            ctx.insert("# nothing!\n" * 10)
+            ctx.finalize()
+            # implicit last finalize inserts [1400, 1450]
+
+        # Check everything
+        eq_(info(), [(1, [100, 145]),
+                     (0, [175, 200]),
+                     (0, [300, 350]),
+                     (0, [400, 450]),
+                     (0, [500, 550]),
+                     (0, [1000, 1050]),
+                     (1, [1100, 1101]),
+                     (1, [1199, 1250]),
+                     (0, [1400, 1450]),
+                     (0, [1500, 1550]),
+                     ])
+
+        # Clean up
+        client.stream_remove("/empty/test")
+        client.stream_destroy("/empty/test")
+        client.close()
+
+    def test_client_12_persistent(self):
+        # Check that connections are persistent when they should be.
+        # This is pretty hard to test; we have to poke deep into
+        # the Requests library.
+        with nilmdb.client.Client(url = testurl) as c:
+            def connections():
+                try:
+                    poolmanager = c.http._last_response.connection.poolmanager
+                    pool = poolmanager.pools[('http','localhost',32180)]
+                    return (pool.num_connections, pool.num_requests)
+                except Exception:
+                    raise SkipTest("can't get connection info")
+
+            # First request makes a connection
+            c.stream_create("/persist/test", "uint16_1")
+            eq_(connections(), (1, 1))
+
+            # Non-generator
+            c.stream_list("/persist/test")
+            eq_(connections(), (1, 2))
+            c.stream_list("/persist/test")
+            eq_(connections(), (1, 3))
+
+            # Generators
+            for x in c.stream_intervals("/persist/test"):
+                pass
+            eq_(connections(), (1, 4))
+            for x in c.stream_intervals("/persist/test"):
+                pass
+            eq_(connections(), (1, 5))
+
+            # Clean up
+            c.stream_remove("/persist/test")
+            c.stream_destroy("/persist/test")
+            eq_(connections(), (1, 7))
+
+    def test_client_13_timestamp_rounding(self):
+        # Test potentially bad timestamps (due to floating point
+        # roundoff etc).  The server will round floating point values
+        # to the nearest int.
+        client = nilmdb.client.Client(testurl)
+
+        client.stream_create("/rounding/test", "uint16_1")
+        with client.stream_insert_context("/rounding/test",
+                                          100000000, 200000000.1) as ctx:
+            ctx.insert("100000000.1 1\n")
+            ctx.insert("150000000.00003 1\n")
+            ctx.insert("199999999.4 1\n")
+        eq_(list(client.stream_intervals("/rounding/test")),
+            [ [ 100000000, 200000000 ] ])
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/rounding/test",
+                                              200000000, 300000000) as ctx:
+                ctx.insert("200000000 1\n")
+                ctx.insert("250000000 1\n")
+                # Server will round this and give an error on finalize()
+                ctx.insert("299999999.99 1\n")
+
+        client.stream_remove("/rounding/test")
+        client.stream_destroy("/rounding/test")
+        client.close()

tests/test_cmdline.py

@@ -1,33 +1,36 @@
 # -*- coding: utf-8 -*-
 
-import nilmdb
+import nilmdb.server
+
 from nilmdb.utils.printf import *
 import nilmdb.cmdline
+from nilmdb.utils import datetime_tz
 
+import unittest
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
-import datetime_tz
 import os
-import shutil
+import re
 import sys
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
 import StringIO
 import shlex
+import warnings
 
-from test_helpers import *
+from testutil.helpers import *
 
 testdb = "tests/cmdline-testdb"
 
-def server_start(max_results = None):
+def server_start(max_results = None, max_removals = None, bulkdata_args = {}):
     global test_server, test_db
     # Start web app on a custom port
-    test_db = nilmdb.NilmDB(testdb, sync = False, max_results = max_results)
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(
+        testdb,
+        max_results = max_results,
+        max_removals = max_removals,
+        bulkdata_args = bulkdata_args)
+    test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
+                                       port = 32180, stoppable = False,
                                        fast_shutdown = True,
                                        force_traceback = False)
     test_server.start(blocking = False)
@@ -59,6 +62,7 @@ class TestCmdline(object):
         passing the given input.  Returns a tuple with the output and
         exit code"""
         # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
+        os.environ['NILMDB_URL'] = "http://localhost:32180/"
         class stdio_wrapper:
             def __init__(self, stdin, stdout, stderr):
                 self.io = (stdin, stdout, stderr)
@@ -94,14 +98,24 @@ class TestCmdline(object):
             self.dump()
             eq_(self.exitcode, 0)
 
-    def fail(self, arg_string, infile = None, exitcode = None):
+    def fail(self, arg_string, infile = None,
+             exitcode = None, require_error = True):
         self.run(arg_string, infile)
         if exitcode is not None and self.exitcode != exitcode:
+            # Wrong exit code
             self.dump()
             eq_(self.exitcode, exitcode)
         if self.exitcode == 0:
+            # Success, when we wanted failure
             self.dump()
             ne_(self.exitcode, 0)
+        # Make sure the output contains the word "error" at the
+        # beginning of a line, but only if an exitcode wasn't
+        # specified.
+        if require_error and not re.search("^error",
+                                           self.captured, re.MULTILINE):
+            raise AssertionError("command failed, but output doesn't "
+                                 "contain the string 'error'")
 
     def contain(self, checkstring):
         in_(checkstring, self.captured)
@@ -114,8 +128,17 @@ class TestCmdline(object):
         with open(file) as f:
             contents = f.read()
             if contents != self.captured:
-                #print contents[1:1000] + "\n"
-                #print self.captured[1:1000] + "\n"
+                print "--- reference file (first 1000 bytes):\n"
+                print contents[0:1000] + "\n"
+                print "--- captured data (first 1000 bytes):\n"
+                print self.captured[0:1000] + "\n"
+                zipped = itertools.izip_longest(contents, self.captured)
+                for (n, (a, b)) in enumerate(zipped):
+                    if a != b:
+                        print "--- first difference is at offset", n
+                        print "--- reference:", repr(a)
+                        print "---  captured:", repr(b)
+                        break
                 raise AssertionError("captured data doesn't match " + file)
 
     def matchfilecount(self, file):
@@ -131,7 +154,7 @@ class TestCmdline(object):
     def dump(self):
         printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
 
-    def test_cmdline_01_basic(self):
+    def test_01_basic(self):
 
         # help
         self.ok("--help")
@@ -148,18 +171,18 @@ class TestCmdline(object):
 
         # try some URL constructions
         self.fail("--url http://nosuchurl/ info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
+        self.contain("error connecting to server")
 
         self.fail("--url nosuchurl info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
+        self.contain("error connecting to server")
 
         self.fail("-u nosuchurl/foo info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
+        self.contain("error connecting to server")
 
-        self.fail("-u localhost:0 info")
-        self.contain("couldn't connect to host")
+        self.fail("-u localhost:1 info")
+        self.contain("error connecting to server")
 
-        self.ok("-u localhost:12380 info")
+        self.ok("-u localhost:32180 info")
         self.ok("info")
 
         # Duplicated arguments should fail, but this isn't implemented
@@ -177,14 +200,55 @@ class TestCmdline(object):
             self.fail("extract --start 2000-01-01 --start 2001-01-02")
             self.contain("duplicated argument")
 
-    def test_cmdline_02_info(self):
+        # Verify that "help command" and "command --help" are identical
+        # for all commands.
+        self.fail("")
+        m = re.search(r"{(.*)}", self.captured)
+        for command in [""] + m.group(1).split(','):
+            self.ok(command + " --help")
+            cap1 = self.captured
+            self.ok("help " + command)
+            cap2 = self.captured
+            self.ok("help " + command + " asdf --url --zxcv -")
+            cap3 = self.captured
+            eq_(cap1, cap2)
+            eq_(cap2, cap3)
+
+    def test_02_parsetime(self):
+        os.environ['TZ'] = "America/New_York"
+        test = datetime_tz.datetime_tz.now()
+        u2ts = nilmdb.utils.time.unix_to_timestamp
+        parse_time = nilmdb.utils.time.parse_time
+        eq_(parse_time(str(test)), u2ts(test.totimestamp()))
+        test = u2ts(datetime_tz.datetime_tz.smartparse("20120405 1400-0400").
+                    totimestamp())
+        eq_(parse_time("hi there 20120405 1400-0400 testing! 123"), test)
+        eq_(parse_time("20120405 1800 UTC"), test)
+        eq_(parse_time("20120405 1400-0400 UTC"), test)
+        for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
+            with assert_raises(ValueError):
+                x = parse_time(badtime)
+        x = parse_time("now")
+        eq_(parse_time("snapshot-20120405-140000.raw.gz"), test)
+        eq_(parse_time("prep-20120405T1400"), test)
+        eq_(parse_time("1333648800.0"), test)
+        eq_(parse_time("1333648800000000"), test)
+        eq_(parse_time("@1333648800000000"), test)
+        eq_(parse_time("min"), nilmdb.utils.time.min_timestamp)
+        eq_(parse_time("max"), nilmdb.utils.time.max_timestamp)
+        with assert_raises(ValueError):
+            parse_time("@hashtag12345")
+
+    def test_03_info(self):
         self.ok("info")
-        self.contain("Server URL: http://localhost:12380/")
+        self.contain("Server URL: http://localhost:32180/")
+        self.contain("Client version: " + nilmdb.__version__)
         self.contain("Server version: " + test_server.version)
         self.contain("Server database path")
         self.contain("Server database size")
+        self.contain("Server database free space")
 
-    def test_cmdline_03_createlist(self):
+    def test_04_createlist(self):
         # Basic stream tests, like those in test_client.
 
         # No streams
@@ -192,57 +256,88 @@ class TestCmdline(object):
         self.match("")
 
         # Bad paths
-        self.fail("create foo/bar/baz PrepData")
+        self.fail("create foo/bar/baz float32_8")
         self.contain("paths must start with /")
 
-        self.fail("create /foo PrepData")
+        self.fail("create /foo float32_8")
         self.contain("invalid path")
+        self.fail("create /newton/prep/ float32_8")
+        self.contain("invalid path")
+
+        self.fail("create /newton/_format/prep float32_8")
+        self.contain("path name is invalid")
+        self.fail("create /_format/newton/prep float32_8")
+        self.contain("path name is invalid")
+        self.fail("create /newton/prep/_format float32_8")
+        self.contain("path name is invalid")
 
         # Bad layout type
         self.fail("create /newton/prep NoSuchLayout")
         self.contain("no such layout")
+        self.fail("create /newton/prep float32_0")
+        self.contain("no such layout")
+        self.fail("create /newton/prep float33_1")
+        self.contain("no such layout")
 
         # Create a few streams
-        self.ok("create /newton/zzz/rawnotch RawNotchedData")
-        self.ok("create /newton/prep PrepData")
-        self.ok("create /newton/raw RawData")
+        self.ok("create /newton/zzz/rawnotch uint16_9")
+        self.ok("create /newton/prep float32_8")
+        self.ok("create /newton/raw uint16_6")
+
+        # Create a stream that already exists
+        self.fail("create /newton/raw uint16_6")
+        self.contain("stream already exists at this path")
 
         # Should not be able to create a stream with another stream as
         # its parent
-        self.fail("create /newton/prep/blah PrepData")
+        self.fail("create /newton/prep/blah float32_8")
         self.contain("path is subdir of existing node")
 
         # Should not be able to create a stream at a location that
         # has other nodes as children
-        self.fail("create /newton/zzz PrepData")
+        self.fail("create /newton/zzz float32_8")
         self.contain("subdirs of this path already exist")
 
         # Verify we got those 3 streams and they're returned in
         # alphabetical order.
         self.ok("list")
-        self.match("/newton/prep PrepData\n"
-                   "/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+        self.match("/newton/prep float32_8\n"
+                   "/newton/raw uint16_6\n"
+                   "/newton/zzz/rawnotch uint16_9\n")
 
-        # Match just one type or one path
+        # Match just one type or one path.  Also check
+        # that --path is optional
         self.ok("list --path /newton/raw")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
-        self.ok("list --layout RawData")
-        self.match("/newton/raw RawData\n")
+        self.ok("list /newton/raw")
+        self.match("/newton/raw uint16_6\n")
+
+        self.fail("list -p /newton/raw /newton/raw")
+        self.contain("too many paths")
+
+        self.ok("list --layout uint16_6")
+        self.match("/newton/raw uint16_6\n")
 
         # Wildcard matches
-        self.ok("list --layout Raw*")
-        self.match("/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+        self.ok("list --layout uint16*")
+        self.match("/newton/raw uint16_6\n"
+                   "/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list --path *zzz* --layout Raw*")
-        self.match("/newton/zzz/rawnotch RawNotchedData\n")
+        self.ok("list --path *zzz* --layout uint16*")
+        self.match("/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list --path *zzz* --layout Prep*")
+        self.ok("list *zzz* --layout uint16*")
+        self.match("/newton/zzz/rawnotch uint16_9\n")
+
+        self.ok("list --path *zzz* --layout float32*")
         self.match("")
 
-    def test_cmdline_04_metadata(self):
+        # reversed range
+        self.fail("list /newton/prep --start 2020-01-01 --end 2000-01-01")
+        self.contain("start must precede end")
+
+    def test_05_metadata(self):
         # Set / get metadata
         self.fail("metadata")
         self.fail("metadata --get")
@@ -299,53 +394,52 @@ class TestCmdline(object):
         self.fail("metadata /newton/nosuchpath")
         self.contain("No stream at path /newton/nosuchpath")
 
-    def test_cmdline_05_parsetime(self):
-        os.environ['TZ'] = "America/New_York"
-        cmd = nilmdb.cmdline.Cmdline(None)
-        test = datetime_tz.datetime_tz.now()
-        eq_(cmd.parse_time(str(test)), test)
-        test = datetime_tz.datetime_tz.smartparse("20120405 1400-0400")
-        eq_(cmd.parse_time("hi there 20120405 1400-0400 testing! 123"), test)
-        eq_(cmd.parse_time("20120405 1800 UTC"), test)
-        eq_(cmd.parse_time("20120405 1400-0400 UTC"), test)
-        for badtime in [ "20120405 1400-9999", "hello", "-", "", "14:00" ]:
-            with assert_raises(ValueError):
-                x = cmd.parse_time(badtime)
-        eq_(cmd.parse_time("snapshot-20120405-140000.raw.gz"), test)
-        eq_(cmd.parse_time("prep-20120405T1400"), test)
-
-    def test_cmdline_06_insert(self):
+    def test_06_insert(self):
         self.ok("insert --help")
 
-        self.fail("insert /foo/bar baz qwer")
-        self.contain("Error getting stream info")
+        self.fail("insert -s 2000 -e 2001 /foo/bar baz")
+        self.contain("error getting stream info")
 
-        self.fail("insert /newton/prep baz qwer")
-        self.match("Error opening input file baz\n")
+        self.fail("insert -s 2000 -e 2001 /newton/prep baz")
+        self.match("error opening input file baz\n")
 
-        self.fail("insert /newton/prep")
-        self.contain("Error extracting time")
+        self.fail("insert /newton/prep --timestamp -f -r 120")
+        self.contain("error extracting start time")
 
-        self.fail("insert --start 19801205 /newton/prep 1 2 3 4")
-        self.contain("--start can only be used with one input file")
+        self.fail("insert /newton/prep --timestamp -r 120")
+        self.contain("need --start or --filename")
 
         self.fail("insert /newton/prep "
                   "tests/data/prep-20120323T1000")
 
+        # insert pre-timestamped data, with bad times (non-monotonic)
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-badtimes") as input:
+            self.fail("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                      input)
+            self.contain("error parsing input data")
+            self.contain("line 7")
+            self.contain("timestamp is not monotonically increasing")
+
         # insert pre-timestamped data, from stdin
         os.environ['TZ'] = "UTC"
         with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
 
         # insert data with normal timestamper from filename
         os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1000 "
+        self.ok("insert --timestamp -f --rate 120 /newton/prep "
+                "tests/data/prep-20120323T1000")
+        self.fail("insert -t --filename /newton/prep "
+                "tests/data/prep-20120323T1002")
+        self.contain("rate is needed")
+        self.ok("insert -t --filename --rate 120 /newton/prep "
                 "tests/data/prep-20120323T1002")
 
         # overlap
         os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/prep "
+        self.fail("insert --timestamp -f --rate 120 /newton/prep "
                   "tests/data/prep-20120323T1004")
         self.contain("overlap")
 
@@ -357,24 +451,33 @@ class TestCmdline(object):
 
         # still an overlap if we specify a different start
         os.environ['TZ'] = "America/New_York"
-        self.fail("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep"
+        self.fail("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep"
                   " tests/data/prep-20120323T1004")
         self.contain("overlap")
 
         # wrong format
         os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/raw "
+        self.fail("insert -t -r 120 -f /newton/raw "
                   "tests/data/prep-20120323T1004")
-        self.contain("Error parsing input data")
+        self.contain("error parsing input data")
+        self.contain("can't parse value")
+
+        # too few rows per line
+        self.ok("create /insert/test float32_20")
+        self.fail("insert -t -r 120 -f /insert/test "
+                  "tests/data/prep-20120323T1004")
+        self.contain("error parsing input data")
+        self.contain("wrong number of values")
+        self.ok("destroy /insert/test")
 
         # empty data does nothing
-        self.ok("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep "
+        self.ok("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep "
                 "/dev/null")
 
         # bad start time
-        self.fail("insert --rate 120 --start 'whatever' /newton/prep /dev/null")
+        self.fail("insert -t -r 120 --start 'whatever' /newton/prep /dev/null")
 
-    def test_cmdline_07_detail(self):
+    def test_07_detail_extended(self):
         # Just count the number of lines, it's probably fine
         self.ok("list --detail")
         lines_(self.captured, 8)
@@ -401,40 +504,75 @@ class TestCmdline(object):
         self.contain("no intervals")
 
         self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'"
-                + " --end='23 Mar 2012 10:05:15.50'")
+                + " --end='23 Mar 2012 10:05:15.51'")
         lines_(self.captured, 2)
         self.contain("10:05:15.500")
 
         self.ok("list --detail")
         lines_(self.captured, 8)
 
-    def test_cmdline_08_extract(self):
+        # Verify the "raw timestamp" output
+        self.ok("list --detail --path *prep --timestamp-raw "
+                "--start='23 Mar 2012 10:05:15.50'")
+        lines_(self.captured, 2)
+        self.contain("[ 1332497115500000 -> 1332497160000000 ]")
+
+        # bad time
+        self.fail("list --detail --path *prep -T --start='9332497115.612'")
+        # good time
+        self.ok("list --detail --path *prep -T --start='1332497115.612'")
+        lines_(self.captured, 2)
+        self.contain("[ 1332497115612000 -> 1332497160000000 ]")
+
+        # Check --ext output
+        self.ok("list --ext")
+        lines_(self.captured, 9)
+
+        self.ok("list -E -T")
+        c = self.contain
+        c("\n  interval extents: 1332496800000000 -> 1332497160000000\n")
+        c("\n        total data: 43200 rows, 359.983336 seconds\n")
+        c("\n  interval extents: (no data)\n")
+        c("\n        total data: 0 rows, 0.000000 seconds\n")
+
+        # Misc
+        self.fail("list --ext --start='23 Mar 2012 10:05:15.50'")
+        self.contain("--start and --end only make sense with --detail")
+
+    def test_08_extract(self):
         # nonexistent stream
         self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")
-        self.contain("Error getting stream info")
+        self.contain("error getting stream info")
 
-        # empty ranges return an error
+        # reversed range
+        self.fail("extract -a /newton/prep --start 2020-01-01 --end 2000-01-01")
+        self.contain("start is after end")
+
+        # empty ranges return error 2
         self.fail("extract -a /newton/prep " +
-                  "--start '23 Mar 2012 10:00:30' " +
-                  "--end '23 Mar 2012 10:00:30'", exitcode = 2)
+                  "--start '23 Mar 2012 20:00:30' " +
+                  "--end '23 Mar 2012 20:00:31'",
+                  exitcode = 2, require_error = False)
         self.contain("no data")
         self.fail("extract -a /newton/prep " +
-                  "--start '23 Mar 2012 10:00:30.000001' " +
-                  "--end '23 Mar 2012 10:00:30.000001'", exitcode = 2)
+                  "--start '23 Mar 2012 20:00:30.000001' " +
+                  "--end '23 Mar 2012 20:00:30.000002'",
+                  exitcode = 2, require_error = False)
         self.contain("no data")
         self.fail("extract -a /newton/prep " +
                   "--start '23 Mar 2022 10:00:30' " +
-                  "--end '23 Mar 2022 10:00:30'", exitcode = 2)
+                  "--end '23 Mar 2022 10:00:31'",
+                  exitcode = 2, require_error = False)
         self.contain("no data")
 
         # but are ok if we're just counting results
         self.ok("extract --count /newton/prep " +
-                "--start '23 Mar 2012 10:00:30' " +
-                "--end '23 Mar 2012 10:00:30'")
+                "--start '23 Mar 2012 20:00:30' " +
+                "--end '23 Mar 2012 20:00:31'")
         self.match("0\n")
         self.ok("extract -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:30.000001' " +
-                "--end '23 Mar 2012 10:00:30.000001'")
+                "--start '23 Mar 2012 20:00:30.000001' " +
+                "--end '23 Mar 2012 20:00:30.000002'")
         self.match("0\n")
 
         # Check various dumps against stored copies of how they should appear
@@ -456,6 +594,10 @@ class TestCmdline(object):
         test(4, "10:00:30.008333", "10:00:30.025")
         test(5, "10:00:30", "10:00:31", extra="--annotate --bare")
         test(6, "10:00:30", "10:00:31", extra="-b")
+        test(7, "10:00:30", "10:00:30.999", extra="-a -T")
+        test(7, "10:00:30", "10:00:30.999", extra="-a --timestamp-raw")
+        test(8, "10:01:59.9", "10:02:00.1", extra="--markup")
+        test(8, "10:01:59.9", "10:02:00.1", extra="-m")
 
         # all data put in by tests
         self.ok("extract -a /newton/prep --start 2000-01-01 --end 2020-01-01")
@@ -463,7 +605,12 @@ class TestCmdline(object):
         self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
         self.match("43200\n")
 
-    def test_cmdline_09_truncated(self):
+        # markup for 3 intervals, plus extra markup lines whenever we had
+        # a "restart" from the nilmdb.stream_extract function
+        self.ok("extract -m /newton/prep --start 2000-01-01 --end 2020-01-01")
+        lines_(self.captured, 43210)
+
+    def test_09_truncated(self):
         # Test truncated responses by overriding the nilmdb max_results
         server_stop()
         server_start(max_results = 2)
@@ -472,7 +619,101 @@ class TestCmdline(object):
         server_stop()
         server_start()
 
-    def test_cmdline_10_destroy(self):
+    def test_10_remove(self):
+        # Removing data
+
+        # Try nonexistent stream
+        self.fail("remove /no/such/foo --start 2000-01-01 --end 2020-01-01")
+        self.contain("No stream at path")
+
+        # empty or backward ranges return errors
+        self.fail("remove /newton/prep --start 2020-01-01 --end 2000-01-01")
+        self.contain("start must precede end")
+
+        self.fail("remove /newton/prep " +
+                  "--start '23 Mar 2012 10:00:30' " +
+                  "--end '23 Mar 2012 10:00:30'")
+        self.contain("start must precede end")
+        self.fail("remove /newton/prep " +
+                  "--start '23 Mar 2012 10:00:30.000001' " +
+                  "--end '23 Mar 2012 10:00:30.000001'")
+        self.contain("start must precede end")
+        self.fail("remove /newton/prep " +
+                  "--start '23 Mar 2022 10:00:30' " +
+                  "--end '23 Mar 2022 10:00:30'")
+        self.contain("start must precede end")
+
+        # Verbose
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2022 20:00:30' " +
+                "--end '23 Mar 2022 20:00:31'")
+        self.match("0\n")
+        self.ok("remove --count /newton/prep " +
+                "--start '23 Mar 2022 20:00:30' " +
+                "--end '23 Mar 2022 20:00:31'")
+        self.match("0\n")
+
+        # Make sure we have the data we expect
+        self.ok("list --detail /newton/prep")
+        self.match("/newton/prep float32_8\n" +
+                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:04:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:06:00.000000 +0000 ]\n")
+
+        # Remove various chunks of prep data and make sure
+        # they're gone.
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:00:30' " +
+                "--end '23 Mar 2012 10:00:40'")
+        self.match("1200\n")
+
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:00:10' " +
+                "--end '23 Mar 2012 10:00:20'")
+        self.match("1200\n")
+
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:00:05' " +
+                "--end '23 Mar 2012 10:00:25'")
+        self.match("1200\n")
+
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:03:50' " +
+                "--end '23 Mar 2012 10:06:50'")
+        self.match("15600\n")
+
+        self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
+        self.match("24000\n")
+
+        # See the missing chunks in list output
+        self.ok("list --detail /newton/prep")
+        self.match("/newton/prep float32_8\n" +
+                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:00:40.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
+
+        # Remove all data, verify it's missing
+        self.ok("remove /newton/prep --start 2000-01-01 --end 2020-01-01")
+        self.match("")  # no count requested this time
+        self.ok("list --detail /newton/prep")
+        self.match("/newton/prep float32_8\n" +
+                   "  (no intervals)\n")
+
+        # Reinsert some data, to verify that no overlaps with deleted
+        # data are reported
+        for minute in ["0", "2"]:
+            self.ok("insert --timestamp -f --rate 120 /newton/prep"
+                    " tests/data/prep-20120323T100" + minute)
+
+    def test_11_destroy(self):
         # Delete records
         self.ok("destroy --help")
 
@@ -482,32 +723,42 @@ class TestCmdline(object):
         self.fail("destroy /no/such/stream")
         self.contain("No stream at path")
 
+        self.fail("destroy -R /no/such/stream")
+        self.contain("No stream at path")
+
         self.fail("destroy asdfasdf")
         self.contain("No stream at path")
 
         # From previous tests, we have:
         self.ok("list")
-        self.match("/newton/prep PrepData\n"
-                   "/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+        self.match("/newton/prep float32_8\n"
+                   "/newton/raw uint16_6\n"
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         # Notice how they're not empty
         self.ok("list --detail")
-        lines_(self.captured, 8)
+        lines_(self.captured, 7)
 
-        # Delete some
-        self.ok("destroy /newton/prep")
+        # Fail to destroy because intervals still present
+        self.fail("destroy /newton/prep")
+        self.contain("all intervals must be removed")
+        self.ok("list --detail")
+        lines_(self.captured, 7)
+
+        # Destroy for real
+        self.ok("destroy -R /newton/prep")
         self.ok("list")
-        self.match("/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+        self.match("/newton/raw uint16_6\n"
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         self.ok("destroy /newton/zzz/rawnotch")
         self.ok("list")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         self.ok("destroy /newton/raw")
-        self.ok("create /newton/raw RawData")
-        self.ok("destroy /newton/raw")
+        self.ok("create /newton/raw uint16_6")
+        # Specify --remove with no data
+        self.ok("destroy --remove /newton/raw")
         self.ok("list")
         self.match("")
 
@@ -516,14 +767,14 @@ class TestCmdline(object):
                     "/newton/raw", "/newton/asdf/qwer" ]
         for path in rebuild:
             # Create the path
-            self.ok("create " + path + " PrepData")
+            self.ok("create " + path + " float32_8")
             self.ok("list")
             self.contain(path)
             # Make sure it was created empty
             self.ok("list --detail --path " + path)
             self.contain("(no intervals)")
 
-    def test_cmdline_11_unicode(self):
+    def test_12_unicode(self):
         # Unicode paths.
         self.ok("destroy /newton/asdf/qwer")
         self.ok("destroy /newton/prep")
@@ -540,3 +791,290 @@ class TestCmdline(object):
         self.ok(u"metadata /düsseldorf/raw --update 'α=β ε τ α'")
         self.ok(u"metadata /düsseldorf/raw")
         self.match(u"α=β ε τ α\nγ=δ\n")
+
+        self.ok(u"destroy /düsseldorf/raw")
+
+    def test_13_files(self):
+        # Test BulkData's ability to split into multiple files,
+        # by forcing the file size to be really small.
+        server_stop()
+        server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
+                                       "files_per_dir" : 3 })
+
+        # Fill data
+        self.ok("create /newton/prep float32_8")
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-timestamped") as input:
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
+
+        # Extract it
+        self.ok("extract /newton/prep --start '2000-01-01' " +
+                "--end '2012-03-23 10:04:01'")
+        lines_(self.captured, 120)
+        self.ok("extract /newton/prep --start '2000-01-01' " +
+                "--end '2022-03-23 10:04:01'")
+        lines_(self.captured, 14400)
+
+        # Make sure there were lots of files generated in the database
+        # dir
+        nfiles = 0
+        for (dirpath, dirnames, filenames) in os.walk(testdb):
+            nfiles += len(filenames)
+        assert(nfiles > 500)
+
+        # Make sure we can restart the server with a different file
+        # size and have it still work
+        server_stop()
+        server_start()
+        self.ok("extract /newton/prep --start '2000-01-01' " +
+                "--end '2022-03-23 10:04:01'")
+        lines_(self.captured, 14400)
+
+        # Now recreate the data one more time and make sure there are
+        # fewer files.
+        self.ok("destroy --remove /newton/prep")
+        self.fail("destroy /newton/prep") # already destroyed
+        self.ok("create /newton/prep float32_8")
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-timestamped") as input:
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
+        nfiles = 0
+        for (dirpath, dirnames, filenames) in os.walk(testdb):
+            nfiles += len(filenames)
+        lt_(nfiles, 50)
+        self.ok("destroy -R /newton/prep") # destroy again
+
+    def test_14_remove_files(self):
+        # Test BulkData's ability to remove when data is split into
+        # multiple files.  Should be a fairly comprehensive test of
+        # remove functionality.
+        # Also limit max_removals, to cover more functionality.
+        server_stop()
+        server_start(max_removals = 4321,
+                     bulkdata_args = { "file_size" : 920, # 23 rows per file
+                                       "files_per_dir" : 3 })
+
+        # Insert data.  Just for fun, insert out of order
+        self.ok("create /newton/prep float32_8")
+        os.environ['TZ'] = "UTC"
+        self.ok("insert -t --filename --rate 120 /newton/prep "
+                "tests/data/prep-20120323T1002")
+        self.ok("insert -t --filename --rate 120 /newton/prep "
+                "tests/data/prep-20120323T1000")
+
+        # Should take up about 2.8 MB here (including directory entries)
+        du_before = nilmdb.utils.diskusage.du(testdb)
+
+        # Make sure we have the data we expect
+        self.ok("list --detail")
+        self.match("/newton/prep float32_8\n" +
+                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n")
+
+        # Remove various chunks of prep data and make sure
+        # they're gone.
+        self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
+        self.match("28800\n")
+
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:00:30' " +
+                "--end '23 Mar 2012 10:03:30'")
+        self.match("21600\n")
+
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:00:10' " +
+                "--end '23 Mar 2012 10:00:20'")
+        self.match("1200\n")
+
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:00:05' " +
+                "--end '23 Mar 2012 10:00:25'")
+        self.match("1200\n")
+
+        self.ok("remove -c /newton/prep " +
+                "--start '23 Mar 2012 10:03:50' " +
+                "--end '23 Mar 2012 10:06:50'")
+        self.match("1200\n")
+
+        self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
+        self.match("3600\n")
+
+        # See the missing chunks in list output
+        self.ok("list --detail")
+        self.match("/newton/prep float32_8\n" +
+                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
+                   "  [ Fri, 23 Mar 2012 10:03:30.000000 +0000"
+                   " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
+
+        # We have 1/8 of the data that we had before, so the file size
+        # should have dropped below 1/4 of what it used to be
+        du_after = nilmdb.utils.diskusage.du(testdb)
+        lt_(du_after, (du_before / 4))
+
+        # Remove anything that came from the 10:02 data file
+        self.ok("remove /newton/prep " +
+                "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
+
+        # Re-insert 19 lines from that file, then remove them again.
+        # With the specific file_size above, this will cause the last
+        # file in the bulk data storage to be exactly file_size large,
+        # so removing the data should also remove that last file.
+        self.ok("insert --timestamp -f --rate 120 /newton/prep " +
+                "tests/data/prep-20120323T1002-first19lines")
+        self.ok("remove /newton/prep " +
+                "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
+
+        # Shut down and restart server, to force nrows to get refreshed.
+        server_stop()
+        server_start()
+
+        # Re-add the full 10:02 data file.  This tests adding new data once
+        # we removed data near the end.
+        self.ok("insert -t -f -r 120 /newton/prep "
+                "tests/data/prep-20120323T1002")
+
+        # See if we can extract it all
+        self.ok("extract /newton/prep --start 2000-01-01 --end 2020-01-01")
+        lines_(self.captured, 15600)
+
+    def test_15_intervals_diff(self):
+        # Test "intervals" and "intervals --diff" command.
+        os.environ['TZ'] = "UTC"
+
+        self.ok("create /diff/1 uint8_1")
+        self.match("")
+        self.ok("intervals /diff/1")
+        self.match("")
+        self.ok("intervals /diff/1 --diff /diff/1")
+        self.match("")
+        self.ok("intervals --diff /diff/1 /diff/1")
+        self.match("")
+        self.fail("intervals /diff/2")
+        self.fail("intervals /diff/1 -d /diff/2")
+
+        self.ok("create /diff/2 uint8_1")
+        self.ok("intervals -T /diff/1 -d /diff/2")
+        self.match("")
+        self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/1 /dev/null")
+
+        self.ok("intervals /diff/1")
+        self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
+                   "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
+
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
+                   "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
+
+        self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/2 /dev/null")
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("")
+
+        self.ok("insert -s 01-01-2001 -e 01-01-2002 /diff/1 /dev/null")
+        self.ok("insert -s 01-01-2002 -e 01-01-2003 /diff/2 /dev/null")
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
+                   "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n")
+
+        self.ok("insert -s 01-01-2004 -e 01-01-2005 /diff/1 /dev/null")
+        self.ok("intervals /diff/1 -d /diff/2")
+        self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
+                   "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n"
+                   "[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
+                   "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
+
+        self.fail("intervals -s 01-01-2003 -e 01-01-2000 /diff/1 -d /diff/2")
+        self.ok("intervals -s 01-01-2003 -e 01-01-2008 /diff/1 -d /diff/2")
+        self.match("[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
+                   "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
+
+        self.ok("destroy -R /diff/1")
+        self.ok("destroy -R /diff/2")
+
+    def test_16_rename(self):
+        # Test renaming.  Force file size smaller so we get more files
+        server_stop()
+        recursive_unlink(testdb)
+        server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
+                                       "files_per_dir" : 3 })
+
+
+        # Fill data
+        self.ok("create /newton/prep float32_8")
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-timestamped") as input:
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
+
+        # Extract it
+        self.ok("extract /newton/prep --start '2000-01-01' " +
+                "--end '2012-03-23 10:04:01'")
+        extract_before = self.captured
+
+        def check_path(*components):
+            # Verify the paths look right on disk
+            seek = os.path.join(testdb, "data", *components)
+            for (dirpath, dirnames, filenames) in os.walk(testdb):
+                if "_format" in filenames:
+                    if dirpath == seek:
+                        break
+                    raise AssertionError("data also found at " + dirpath)
+            else:
+                raise AssertionError("data not found at " + seek)
+            # Verify "list" output
+            self.ok("list")
+            self.match("/" + "/".join(components) + " float32_8\n")
+
+        # Lots of renames
+        check_path("newton", "prep")
+
+        self.fail("rename /newton/prep /newton/prep")
+        self.contain("old and new paths are the same")
+        check_path("newton", "prep")
+        self.fail("rename /newton/prep /newton")
+        self.contain("subdirs of this path already exist")
+        self.fail("rename /newton/prep /newton/prep/")
+        self.contain("invalid path")
+        self.ok("rename /newton/prep /newton/foo")
+        check_path("newton", "foo")
+        self.ok("rename /newton/foo /totally/different/thing")
+        check_path("totally", "different", "thing")
+        self.ok("rename /totally/different/thing /totally/something")
+        check_path("totally", "something")
+        self.ok("rename /totally/something /totally/something/cool")
+        check_path("totally", "something", "cool")
+        self.ok("rename /totally/something/cool /foo/bar")
+        check_path("foo", "bar")
+        self.ok("create /xxx/yyy/zzz float32_8")
+        self.fail("rename /foo/bar /xxx/yyy")
+        self.contain("subdirs of this path already exist")
+        self.fail("rename /foo/bar /xxx/yyy/zzz")
+        self.contain("stream already exists at this path")
+        self.fail("rename /foo/bar /xxx/yyy/zzz/www")
+        self.contain("path is subdir of existing node")
+        self.ok("rename /foo/bar /xxx/yyy/mmm")
+        self.ok("destroy -R /xxx/yyy/zzz")
+        check_path("xxx", "yyy", "mmm")
+
+        # Extract it at the final path
+        self.ok("extract /xxx/yyy/mmm --start '2000-01-01' " +
+                "--end '2012-03-23 10:04:01'")
+        eq_(self.captured, extract_before)
+
+        self.ok("destroy -R /xxx/yyy/mmm")
+
+        # Make sure temporary rename dirs weren't left around
+        for (dirpath, dirnames, filenames) in os.walk(testdb):
+            if "rename-" in dirpath:
+                raise AssertionError("temporary directories not cleaned up")
+            if "totally" in dirpath or "newton" in dirpath:
+                raise AssertionError("old directories not cleaned up")
+
+        server_stop()
+        server_start()

tests/test_interval.py

@@ -2,21 +2,25 @@
 
 import nilmdb
 from nilmdb.utils.printf import *
-import datetime_tz
+from nilmdb.utils import datetime_tz
 
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
 
-from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
+from nilmdb.utils.interval import IntervalError
+from nilmdb.server.interval import Interval, DBInterval, IntervalSet
 
-from test_helpers import *
+# so we can test them separately
+from nilmdb.utils.interval import Interval as UtilsInterval
+
+from testutil.helpers import *
 import unittest
 
 # set to False to skip live renders
 do_live_renders = False
 def render(iset, description = "", live = True):
-    import renderdot
+    import testutil.renderdot as renderdot
     r = renderdot.RBTreeRenderer(iset.tree)
     return r.render(description, live and do_live_renders)
 
@@ -46,15 +50,24 @@ def makeset(string):
     return iset
 
 class TestInterval:
+    def test_client_interval(self):
+        # Run interval tests against the Python version of Interval.
+        global Interval
+        NilmdbInterval = Interval
+        Interval = UtilsInterval
+        self.test_interval()
+        self.test_interval_intersect()
+        Interval = NilmdbInterval
+
     def test_interval(self):
         # Test Interval class
         os.environ['TZ'] = "America/New_York"
         datetime_tz._localtz = None
-        (d1, d2, d3) = [ datetime_tz.datetime_tz.smartparse(x).totimestamp()
+        (d1, d2, d3) = [ nilmdb.utils.time.parse_time(x)
                          for x in [ "03/24/2012", "03/25/2012", "03/26/2012" ] ]
 
         # basic construction
-        i = Interval(d1, d1)
+        i = Interval(d1, d2)
         i = Interval(d1, d3)
         eq_(i.start, d1)
         eq_(i.end, d3)
@@ -76,8 +89,8 @@ class TestInterval:
         assert(Interval(d1, d3) > Interval(d1, d2))
         assert(Interval(d1, d2) < Interval(d2, d3))
         assert(Interval(d1, d3) < Interval(d2, d3))
-        assert(Interval(d2, d2) > Interval(d1, d3))
-        assert(Interval(d3, d3) == Interval(d3, d3))
+        assert(Interval(d2, d2+1) > Interval(d1, d3))
+        assert(Interval(d3, d3+1) == Interval(d3, d3+1))
         #with assert_raises(TypeError): # was AttributeError, that's wrong
         #    x = (i == 123)
 
@@ -86,16 +99,16 @@ class TestInterval:
         with assert_raises(IntervalError):
             x = Interval(d2, d3).subset(d1, d2)
 
-        # big integers and floats
-        x = Interval(5000111222, 6000111222)
-        eq_(str(x), "[5000111222.0 -> 6000111222.0)")
-        x = Interval(123.45, 234.56)
-        eq_(str(x), "[123.45 -> 234.56)")
+        # big integers, negative integers
+        x = Interval(5000111222000000, 6000111222000000)
+        eq_(str(x), "[5000111222000000 -> 6000111222000000)")
+        x = Interval(-5000111222000000, -4000111222000000)
+        eq_(str(x), "[-5000111222000000 -> -4000111222000000)")
 
         # misc
         i = Interval(d1, d2)
         eq_(repr(i), repr(eval(repr(i))))
-        eq_(str(i), "[1332561600.0 -> 1332648000.0)")
+        eq_(str(i), "[1332561600000000 -> 1332648000000000)")
 
     def test_interval_intersect(self):
         # Test Interval intersections
@@ -191,7 +204,8 @@ class TestInterval:
 
         # misc
         eq_(repr(iset), repr(eval(repr(iset))))
-        eq_(str(iset), "[[100.0 -> 200.0), [200.0 -> 300.0)]")
+        eq_(str(iset),
+            "[[100 -> 200), [200 -> 300)]")
 
     def test_intervalset_geniset(self):
         # Test basic iset construction
@@ -206,64 +220,90 @@ class TestInterval:
             makeset("  [-|-----|"))
 
 
-    def test_intervalset_intersect(self):
+    def test_intervalset_intersect_difference(self):
         # Test intersection (&)
         with assert_raises(TypeError): # was AttributeError
             x = makeset("[--)") & 1234
 
-        # Intersection with interval
-        eq_(makeset("[---|---)[)") &
-            list(makeset("  [------) "))[0],
-            makeset("  [-----)  "))
+        def do_test(a, b, c, d):
+            # a & b == c
+            ab = IntervalSet()
+            for x in b:
+                for i in (a & x):
+                    ab += i
+            eq_(ab,c)
 
-        # Intersection with sets
-        eq_(makeset("[---------)") &
-            makeset(" [---)     "),
-            makeset(" [---)     "))
+            # a \ b == d
+            eq_(IntervalSet(nilmdb.utils.interval.set_difference(a,b)), d)
 
-        eq_(makeset(" [---)     ") &
-            makeset("[---------)"),
-            makeset(" [---)     "))
-
-        eq_(makeset("    [-----)") &
-            makeset(" [-----)   "),
-            makeset("    [--)   "))
-
-        eq_(makeset(" [--)  [--)") &
+        # Intersection with intervals
+        do_test(makeset("[---|---)[)"),
                 makeset("  [------) "),
-            makeset("  [-)  [-) "))
+                makeset("  [-----)  "), # intersection
+                makeset("[-)      [)")) # difference
 
-        eq_(makeset("      [---)") &
+        do_test(makeset("[---------)"),
+                makeset(" [---)     "),
+                makeset(" [---)     "), # intersection
+                makeset("[)   [----)")) # difference
+
+        do_test(makeset(" [---)     "),
+                makeset("[---------)"),
+                makeset(" [---)     "), # intersection
+                makeset("           ")) # difference
+
+        do_test(makeset("    [-----)"),
+                makeset(" [-----)   "),
+                makeset("    [--)   "), # intersection
+                makeset("       [--)")) # difference
+
+        do_test(makeset(" [--)  [--)"),
+                makeset("  [------) "),
+                makeset("  [-)  [-) "), # intersection
+                makeset(" [)      [)")) # difference
+
+        do_test(makeset("      [---)"),
                 makeset(" [--)      "),
-            makeset("           "))
+                makeset("           "), # intersection
+                makeset("      [---)")) # difference
 
-        eq_(makeset("    [-|---)") &
+        do_test(makeset("    [-|---)"),
                 makeset(" [-----|-) "),
-            makeset("    [----) "))
+                makeset("    [----) "), # intersection
+                makeset("         [)")) # difference
 
-        eq_(makeset("    [-|-)  ") &
+        do_test(makeset("    [-|-)  "),
                 makeset(" [-|--|--) "),
-            makeset("    [---)  "))
+                makeset("    [---)  "), # intersection
+                makeset("           ")) # difference
+
+        do_test(makeset("[-)[-)[-)[)"),
+                makeset(" [)  [|)[) "),
+                makeset(" [)   [)   "), # intersection
+                makeset("[) [-) [)[)")) # difference
 
         # Border cases -- will give different results if intervals are
-        # half open or fully closed.  Right now, they are half open,
-        # although that's a little messy since the database intervals
-        # often contain a data point at the endpoint.
-        half_open = True
-        if half_open:
-            eq_(makeset("      [---)") &
+        # half open or fully closed.  In nilmdb, they are half open.
+        do_test(makeset("      [---)"),
                 makeset(" [----)    "),
-                makeset("           "))
-            eq_(makeset(" [----)[--)") &
+                makeset("           "), # intersection
+                makeset("      [---)")) # difference
+
+        do_test(makeset(" [----)[--)"),
                 makeset("[-) [--) [)"),
-                makeset(" [) [-)  [)"))
-        else:
-            eq_(makeset("      [---)") &
-                makeset(" [----)    "),
-                makeset("      .    "))
-            eq_(makeset(" [----)[--)") &
-                makeset("[-) [--) [)"),
-                makeset(" [) [-). [)"))
+                makeset(" [) [-)  [)"), # intersection
+                makeset("  [-)  [-) ")) # difference
+
+        # Set difference with bounds
+        a = makeset(" [----)[--)")
+        b = makeset("[-) [--) [)")
+        c = makeset("[----)     ")
+        d = makeset("  [-)      ")
+        eq_(nilmdb.utils.interval.set_difference(
+            a.intersection(list(c)[0]), b.intersection(list(c)[0])), d)
+
+        # Empty second set
+        eq_(nilmdb.utils.interval.set_difference(a, IntervalSet()), a)
 
 class TestIntervalDB:
     def test_dbinterval(self):
@@ -292,7 +332,7 @@ class TestIntervalDB:
         # actual start, end can be a subset
         a = DBInterval(150, 200, 100, 200, 10000, 20000)
         b = DBInterval(100, 150, 100, 200, 10000, 20000)
-        c = DBInterval(150, 150, 100, 200, 10000, 20000)
+        c = DBInterval(150, 160, 100, 200, 10000, 20000)
 
         # Make a set of DBIntervals
         iseta = IntervalSet([a, b])
@@ -345,14 +385,14 @@ class TestIntervalSpeed:
     def test_interval_speed(self):
         import yappi
         import time
-        import aplotter
         import random
         import math
 
         print
         yappi.start()
         speeds = {}
-        for j in [ 2**x for x in range(5,20) ]:
+        limit = 22 # was 20
+        for j in [ 2**x for x in range(5,limit) ]:
             start = time.time()
             iset = IntervalSet()
             for i in random.sample(xrange(j),j):
@@ -365,7 +405,5 @@ class TestIntervalSpeed:
                    speed/j,
                    speed / (j*math.log(j))) # should be constant
             speeds[j] = speed
-        aplotter.plot(speeds.keys(), speeds.values(), plot_slope=True)
         yappi.stop()
         yappi.print_stats(sort_type=yappi.SORTTYPE_TTOT, limit=10)
-

tests/test_iteratorizer.py

@@ -1,55 +0,0 @@
-import nilmdb
-from nilmdb.utils.printf import *
-
-import nose
-from nose.tools import *
-from nose.tools import assert_raises
-import threading
-import time
-
-from test_helpers import *
-
-def func_with_callback(a, b, callback):
-    callback(a)
-    callback(b)
-    callback(a+b)
-
-class TestIteratorizer(object):
-    def test(self):
-
-        # First try it with a normal callback
-        self.result = ""
-        def cb(x):
-            self.result += str(x)
-        func_with_callback(1, 2, cb)
-        eq_(self.result, "123")
-
-        # Now make it an iterator
-        it = nilmdb.utils.Iteratorizer(
-            lambda x:
-            func_with_callback(1, 2, x))
-        result = ""
-        for i in it:
-            result += str(i)
-        eq_(result, "123")
-
-        # Make sure things work when an exception occurs
-        it = nilmdb.utils.Iteratorizer(
-            lambda x:
-            func_with_callback(1, "a", x))
-        result = ""
-        with assert_raises(TypeError) as e:
-            for i in it:
-                result += str(i)
-        eq_(result, "1a")
-
-        # Now try to trigger the case where we stop iterating
-        # mid-generator, and expect the iteratorizer to clean up after
-        # itself.  This doesn't have a particular result in the test,
-        # but gains coverage.
-        def foo():
-            it = nilmdb.utils.Iteratorizer(
-                lambda x:
-                func_with_callback(1, 2, x))
-            it.next()
-        foo()

tests/test_layout.py

@@ -1,254 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import nilmdb
-
-from nilmdb.utils.printf import *
-
-from nose.tools import *
-from nose.tools import assert_raises
-import distutils.version
-import itertools
-import os
-import shutil
-import sys
-import cherrypy
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
-import cStringIO
-import random
-import unittest
-
-from test_helpers import *
-
-from nilmdb.layout import *
-
-class TestLayouts(object):
-    # Some nilmdb.layout tests.  Not complete, just fills in missing
-    # coverage.
-    def test_layouts(self):
-        x = nilmdb.layout.get_named("PrepData")
-        y = nilmdb.layout.get_named("float32_8")
-        eq_(x.count, y.count)
-        eq_(x.datatype, y.datatype)
-        y = nilmdb.layout.get_named("float32_7")
-        ne_(x.count, y.count)
-        eq_(x.datatype, y.datatype)
-
-    def test_parsing(self):
-        self.real_t_parsing("PrepData", "RawData", "RawNotchedData")
-        self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
-    def real_t_parsing(self, name_prep, name_raw, name_rawnotch):
-        # invalid layouts
-        with assert_raises(TypeError) as e:
-            parser = Parser("NoSuchLayout")
-        with assert_raises(TypeError) as e:
-            parser = Parser("float32")
-
-        # too little data
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5\n")
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("error", str(e.exception))
-
-        # too much data
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n")
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("error", str(e.exception))
-
-        # just right
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n")
-        parser.parse(data)
-        eq_(parser.min_timestamp, 1234567890.0)
-        eq_(parser.max_timestamp, 1234567890.1)
-        eq_(parser.data, [[1234567890.0,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8],
-                          [1234567890.1,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]])
-
-        # try RawData too, with clamping
-        parser = Parser(name_raw)
-        data = ( "1234567890.000000 1 2 3 4 5 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        parser.parse(data)
-        eq_(parser.data, [[1234567890.0,1,2,3,4,5,6],
-                          [1234567890.1,1,2,3,4,5,6]])
-
-        # pass an instantiated class
-        parser = Parser(get_named(name_rawnotch))
-        data = ( "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
-                 "1234567890.100000 1 2 3 4 5 6 7 8 9\n" )
-        parser.parse(data)
-
-        # non-monotonic
-        parser = Parser(name_raw)
-        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.000000 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("not monotonically increasing", str(e.exception))
-
-        # RawData with values out of bounds
-        parser = Parser(name_raw)
-        data = ( "1234567890.000000 1 2 3 4 500000 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("value out of range", str(e.exception))
-
-        # Empty data should work but is useless
-        parser = Parser(name_raw)
-        data = ""
-        parser.parse(data)
-        assert(parser.min_timestamp is None)
-        assert(parser.max_timestamp is None)
-
-    def test_formatting(self):
-        self.real_t_formatting("PrepData", "RawData", "RawNotchedData")
-        self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
-    def real_t_formatting(self, name_prep, name_raw, name_rawnotch):
-        # invalid layout
-        with assert_raises(TypeError) as e:
-            formatter = Formatter("NoSuchLayout")
-
-        # too little data
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5 ],
-                 [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5 ] ]
-        with assert_raises(FormatterError) as e:
-            formatter.format(data)
-        in_("error", str(e.exception))
-
-        # too much data
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
-        with assert_raises(FormatterError) as e:
-            formatter.format(data)
-        in_("error", str(e.exception))
-
-        # just right
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ],
-                 [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1.100000 2.200000 3.300000 4.400000 " +
-            "5.500000 6.600000 7.700000 8.800000\n" +
-            "1234567890.100000 1.100000 2.200000 3.300000 4.400000 " +
-            "5.500000 6.600000 7.700000 8.800000\n")
-
-        # try RawData too
-        formatter = Formatter(name_raw)
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1 2 3 4 5 6\n" +
-            "1234567890.100000 1 2 3 4 5 6\n")
-
-        # pass an instantiated class
-        formatter = Formatter(get_named(name_rawnotch))
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
-            "1234567890.100000 1 2 3 4 5 6 7 8 9\n")
-
-        # Empty data should work but is useless
-        formatter = Formatter(name_raw)
-        data = []
-        text = formatter.format(data)
-        eq_(text, "")
-
-    def test_roundtrip(self):
-        self.real_t_roundtrip("PrepData", "RawData", "RawNotchedData")
-        self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
-    def real_t_roundtrip(self, name_prep, name_raw, name_rawnotch):
-        # Verify that textual data passed into the Parser, and then
-        # back through the Formatter, then back into the Parser,
-        # gives identical parsed representations
-        random.seed(12345)
-
-        def do_roundtrip(layout, datagen):
-            for i in range(100):
-                rows = random.randint(1,100)
-                data = ""
-                ts = 1234567890
-                for r in range(rows):
-                    ts += random.uniform(0,1)
-                    row = sprintf("%f", ts) + " "
-                    row += " ".join(datagen())
-                    row += "\n"
-                    data += row
-                parser1 = Parser(layout)
-                formatter = Formatter(layout)
-                parser2 = Parser(layout)
-                parser1.parse(data)
-                parser2.parse(formatter.format(parser1.data))
-                eq_(parser1.data, parser2.data)
-
-        def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
-                     for x in range(8) ]
-        do_roundtrip(name_prep, datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(6) ]
-        do_roundtrip(name_raw, datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(9) ]
-        do_roundtrip(name_rawnotch, datagen)
-
-class TestLayoutSpeed:
-    @unittest.skip("this is slow")
-    def test_layout_speed(self):
-        import time
-
-        random.seed(54321)
-
-        def do_speedtest(layout, datagen, rows = 5000, times = 100):
-            # Build data once
-            data = ""
-            ts = 1234567890
-            for r in range(rows):
-                ts += random.uniform(0,1)
-                row = sprintf("%f", ts) + " "
-                row += " ".join(datagen())
-                row += "\n"
-                data += row
-
-            # Do lots of roundtrips
-            start = time.time()
-            for i in range(times):
-                parser = Parser(layout)
-                formatter = Formatter(layout)
-                parser.parse(data)
-                data = formatter.format(parser.data)
-            elapsed = time.time() - start
-            printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
-                   layout,
-                   elapsed * 1e3,
-                   (elapsed * 1e6) / (rows * times),
-                   (rows * times) / elapsed)
-
-        print ""
-        def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
-                     for x in range(10) ]
-        do_speedtest("float32_10", datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(10) ]
-        do_speedtest("uint16_10", datagen)

tests/test_lrucache.py

@@ -6,8 +6,9 @@ from nose.tools import *
 from nose.tools import assert_raises
 import threading
 import time
+import inspect
 
-from test_helpers import *
+from testutil.helpers import *
 
 @nilmdb.utils.lru_cache(size = 3)
 def foo1(n):
@@ -24,30 +25,59 @@ foo3d.destructed = []
 def foo3(n):
     return n
 
+class Foo:
+    def __init__(self):
+        self.calls = 0
+    @nilmdb.utils.lru_cache(size = 3, keys = slice(1, 2))
+    def foo(self, n, **kwargs):
+        self.calls += 1
+
 class TestLRUCache(object):
     def test(self):
+
         [ foo1(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_((foo1.cache_hits, foo1.cache_misses), (6, 3))
+        eq_(foo1.cache_info(), (6, 3))
         [ foo1(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_((foo1.cache_hits, foo1.cache_misses), (15, 3))
+        eq_(foo1.cache_info(), (15, 3))
         [ foo1(n) for n in [ 4, 2, 1, 1, 4 ] ]
-        eq_((foo1.cache_hits, foo1.cache_misses), (18, 5))
+        eq_(foo1.cache_info(), (18, 5))
 
         [ foo2(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_((foo2.cache_hits, foo2.cache_misses), (6, 3))
+        eq_(foo2.cache_info(), (6, 3))
         [ foo2(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_((foo2.cache_hits, foo2.cache_misses), (15, 3))
+        eq_(foo2.cache_info(), (15, 3))
         [ foo2(n) for n in [ 4, 2, 1, 1, 4 ] ]
-        eq_((foo2.cache_hits, foo2.cache_misses), (19, 4))
+        eq_(foo2.cache_info(), (19, 4))
 
         [ foo3(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_((foo3.cache_hits, foo3.cache_misses), (6, 3))
+        eq_(foo3.cache_info(), (6, 3))
         [ foo3(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_((foo3.cache_hits, foo3.cache_misses), (15, 3))
+        eq_(foo3.cache_info(), (15, 3))
         [ foo3(n) for n in [ 4, 2, 1, 1, 4 ] ]
-        eq_((foo3.cache_hits, foo3.cache_misses), (18, 5))
+        eq_(foo3.cache_info(), (18, 5))
         eq_(foo3d.destructed, [1, 3])
+        with assert_raises(KeyError):
+            foo3.cache_remove(1,2,3)
         foo3.cache_remove(1)
         eq_(foo3d.destructed, [1, 3, 1])
         foo3.cache_remove_all()
         eq_(foo3d.destructed, [1, 3, 1, 2, 4 ])
+
+        foo = Foo()
+        foo.foo(5)
+        foo.foo(6)
+        foo.foo(7)
+        foo.foo(5)
+        eq_(foo.calls, 3)
+
+        # Can't handle keyword arguments right now
+        with assert_raises(NotImplementedError):
+            foo.foo(3, asdf = 7)
+
+        # Verify that argspecs were maintained
+        eq_(inspect.getargspec(foo1),
+            inspect.ArgSpec(args=['n'],
+                            varargs=None, keywords=None, defaults=None))
+        eq_(inspect.getargspec(foo.foo),
+            inspect.ArgSpec(args=['self', 'n'],
+                            varargs=None, keywords="kwargs", defaults=None))

tests/test_mustclose.py

@@ -5,17 +5,20 @@ import nose
 from nose.tools import *
 from nose.tools import assert_raises
 
-from test_helpers import *
+from testutil.helpers import *
 
 import sys
 import cStringIO
+import gc
+
+import inspect
 
 err = cStringIO.StringIO()
 
 @nilmdb.utils.must_close(errorfile = err)
 class Foo:
-    def __init__(self):
-        fprintf(err, "Init\n")
+    def __init__(self, arg):
+        fprintf(err, "Init %s\n", arg)
 
     def __del__(self):
         fprintf(err, "Deleting\n")
@@ -23,8 +26,26 @@ class Foo:
     def close(self):
         fprintf(err, "Closing\n")
 
-@nilmdb.utils.must_close(errorfile = err)
+@nilmdb.utils.must_close(errorfile = err, wrap_verify = True)
 class Bar:
+    def __init__(self):
+        fprintf(err, "Init\n")
+
+    def __del__(self):
+        fprintf(err, "Deleting\n")
+
+    @classmethod
+    def baz(self):
+        fprintf(err, "Baz\n")
+
+    def close(self):
+        fprintf(err, "Closing\n")
+
+    def blah(self, arg):
+        fprintf(err, "Blah %s\n", arg)
+
+@nilmdb.utils.must_close(errorfile = err)
+class Baz:
     pass
 
 class TestMustClose(object):
@@ -34,26 +55,60 @@ class TestMustClose(object):
         # garbage collect the object (and call its __del__ function)
         # right after a "del x".
 
-        x = Foo()
+        # Trigger error
+        err.truncate()
+        x = Foo("hi")
+        # Verify that the arg spec was maintained
+        eq_(inspect.getargspec(x.__init__),
+            inspect.ArgSpec(args = ['self', 'arg'],
+                            varargs = None, keywords = None, defaults = None))
         del x
+        gc.collect()
         eq_(err.getvalue(),
-            "Init\n"
+            "Init hi\n"
             "error: Foo.close() wasn't called!\n"
             "Deleting\n")
 
+        # No error
         err.truncate(0)
-
-        y = Foo()
+        y = Foo("bye")
         y.close()
         del y
+        gc.collect()
         eq_(err.getvalue(),
-            "Init\n"
+            "Init bye\n"
             "Closing\n"
             "Deleting\n")
 
+        # Verify function calls when wrap_verify is True
         err.truncate(0)
-
         z = Bar()
+        eq_(inspect.getargspec(z.blah),
+            inspect.ArgSpec(args = ['self', 'arg'],
+                            varargs = None, keywords = None, defaults = None))
+        z.blah("boo")
         z.close()
+        with assert_raises(AssertionError) as e:
+            z.blah("hello")
+        in_("called <function blah at 0x", str(e.exception))
+        in_("> after close", str(e.exception))
+        # Since the most recent assertion references 'z',
+        # we need to raise another assertion here so that
+        # 'z' will get properly deleted.
+        with assert_raises(AssertionError):
+            raise AssertionError()
         del z
+        gc.collect()
+        eq_(err.getvalue(),
+            "Init\n"
+            "Blah boo\n"
+            "Closing\n"
+            "Deleting\n")
+
+        # Class with missing methods
+        err.truncate(0)
+        w = Baz()
+        w.close()
+        del w
         eq_(err.getvalue(), "")
+

tests/test_nilmdb.py

@@ -1,4 +1,4 @@
-import nilmdb
+import nilmdb.server
 
 from nose.tools import *
 from nose.tools import assert_raises
@@ -6,15 +6,15 @@ import distutils.version
 import simplejson as json
 import itertools
 import os
-import shutil
 import sys
-import cherrypy
 import threading
 import urllib2
 from urllib2 import urlopen, HTTPError
-import Queue
 import cStringIO
 import time
+import requests
+
+from nilmdb.utils import serializer_proxy
 
 testdb = "tests/testdb"
 
@@ -22,18 +22,15 @@ testdb = "tests/testdb"
 #def cleanup():
 #    os.unlink(testdb)
 
-from test_helpers import *
+from testutil.helpers import *
 
 class Test00Nilmdb(object):  # named 00 so it runs first
     def test_NilmDB(self):
         recursive_unlink(testdb)
 
-        with assert_raises(IOError):
-            nilmdb.NilmDB("/nonexistant-db/foo")
-
-        db = nilmdb.NilmDB(testdb)
+        db = nilmdb.server.NilmDB(testdb)
         db.close()
-        db = nilmdb.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         db.close()
 
         # test timer, just to get coverage
@@ -46,29 +43,29 @@ class Test00Nilmdb(object):  # named 00 so it runs first
         in_("test: ", capture.getvalue())
 
     def test_stream(self):
-        db = nilmdb.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         eq_(db.stream_list(), [])
 
         # Bad path
         with assert_raises(ValueError):
-            db.stream_create("foo/bar/baz", "PrepData")
+            db.stream_create("foo/bar/baz", "float32_8")
         with assert_raises(ValueError):
-            db.stream_create("/foo", "PrepData")
+            db.stream_create("/foo", "float32_8")
         # Bad layout type
         with assert_raises(ValueError):
             db.stream_create("/newton/prep", "NoSuchLayout")
-        db.stream_create("/newton/prep", "PrepData")
-        db.stream_create("/newton/raw", "RawData")
-        db.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
+        db.stream_create("/newton/prep", "float32_8")
+        db.stream_create("/newton/raw", "uint16_6")
+        db.stream_create("/newton/zzz/rawnotch", "uint16_9")
 
         # Verify we got 3 streams
-        eq_(db.stream_list(), [ ["/newton/prep", "PrepData"],
-                                ["/newton/raw", "RawData"],
-                                ["/newton/zzz/rawnotch", "RawNotchedData"]
+        eq_(db.stream_list(), [ ["/newton/prep", "float32_8"],
+                                ["/newton/raw", "uint16_6"],
+                                ["/newton/zzz/rawnotch", "uint16_9"]
                                 ])
         # Match just one type or one path
-        eq_(db.stream_list(layout="RawData"), [ ["/newton/raw", "RawData"] ])
-        eq_(db.stream_list(path="/newton/raw"), [ ["/newton/raw", "RawData"] ])
+        eq_(db.stream_list(layout="uint16_6"), [ ["/newton/raw", "uint16_6"] ])
+        eq_(db.stream_list(path="/newton/raw"), [ ["/newton/raw", "uint16_6"] ])
 
         # Verify that columns were made right (pytables specific)
         if "h5file" in db.data.__dict__:
@@ -93,19 +90,31 @@ class Test00Nilmdb(object):  # named 00 so it runs first
         eq_(db.stream_get_metadata("/newton/prep"), meta1)
         eq_(db.stream_get_metadata("/newton/raw"), meta1)
 
+        # fill in some test coverage for start >= end
+        with assert_raises(nilmdb.server.NilmDBError):
+            db.stream_remove("/newton/prep", 0, 0)
+        with assert_raises(nilmdb.server.NilmDBError):
+            db.stream_remove("/newton/prep", 1, 0)
+        db.stream_remove("/newton/prep", 0, 1)
+
         db.close()
 
 class TestBlockingServer(object):
     def setUp(self):
-        self.db = nilmdb.NilmDB(testdb, sync=False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
 
     def tearDown(self):
         self.db.close()
 
     def test_blocking_server(self):
+        # Server should fail if the database doesn't have a "_thread_safe"
+        # property.
+        with assert_raises(KeyError):
+            nilmdb.server.Server(object())
+
         # Start web app on a custom port
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = True)
+        self.server = nilmdb.server.Server(self.db, host = "127.0.0.1",
+                                           port = 32180, stoppable = True)
 
         # Run it
         event = threading.Event()
@@ -113,16 +122,17 @@ class TestBlockingServer(object):
             self.server.start(blocking = True, event = event)
         thread = threading.Thread(target = run_server)
         thread.start()
-        event.wait(timeout = 2)
+        if not event.wait(timeout = 10):
+            raise AssertionError("server didn't start in 10 seconds")
 
         # Send request to exit.
-        req = urlopen("http://127.0.0.1:12380/exit/", timeout = 1)
+        req = urlopen("http://127.0.0.1:32180/exit/", timeout = 1)
 
         # Wait for it
         thread.join()
 
 def geturl(path):
-    req = urlopen("http://127.0.0.1:12380" + path, timeout = 10)
+    req = urlopen("http://127.0.0.1:32180" + path, timeout = 10)
     return req.read()
 
 def getjson(path):
@@ -132,9 +142,9 @@ class TestServer(object):
 
     def setUp(self):
         # Start web app on a custom port
-        self.db = nilmdb.NilmDB(testdb, sync=False)
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
+        self.server = nilmdb.server.Server(self.db, host = "127.0.0.1",
+                                           port = 32180, stoppable = False)
         self.server.start(blocking = False)
 
     def tearDown(self):
@@ -150,21 +160,21 @@ class TestServer(object):
             eq_(e.exception.code, 404)
 
         # Check version
-        eq_(distutils.version.StrictVersion(getjson("/version")),
-            distutils.version.StrictVersion(self.server.version))
+        eq_(distutils.version.LooseVersion(getjson("/version")),
+            distutils.version.LooseVersion(nilmdb.__version__))
 
     def test_stream_list(self):
         # Known streams that got populated by an earlier test (test_nilmdb)
         streams = getjson("/stream/list")
 
         eq_(streams, [
-            ['/newton/prep', 'PrepData'],
-            ['/newton/raw', 'RawData'],
-            ['/newton/zzz/rawnotch', 'RawNotchedData'],
+            ['/newton/prep', 'float32_8'],
+            ['/newton/raw', 'uint16_6'],
+            ['/newton/zzz/rawnotch', 'uint16_9'],
             ])
 
-        streams = getjson("/stream/list?layout=RawData")
-        eq_(streams, [['/newton/raw', 'RawData']])
+        streams = getjson("/stream/list?layout=uint16_6")
+        eq_(streams, [['/newton/raw', 'uint16_6']])
 
         streams = getjson("/stream/list?layout=NoSuchLayout")
         eq_(streams, [])
@@ -194,11 +204,50 @@ class TestServer(object):
                        "&key=foo")
         eq_(data, {'foo': None})
 
+    def test_cors_headers(self):
+        # Test that CORS headers are being set correctly
 
-    def test_insert(self):
-        # GET instead of POST (no body)
-        # (actual POST test is done by client code)
-        with assert_raises(HTTPError) as e:
-            getjson("/stream/insert?path=/newton/prep&start=0&end=0")
-        eq_(e.exception.code, 400)
+        # Normal GET should send simple response
+        url = "http://127.0.0.1:32180/stream/list"
+        r = requests.get(url, headers = { "Origin": "http://google.com/" })
+        eq_(r.status_code, 200)
+        if "access-control-allow-origin" not in r.headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-origin"], "http://google.com/")
 
+        # OPTIONS without CORS preflight headers should result in 405
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            })
+        eq_(r.status_code, 405)
+
+        # OPTIONS with preflight headers should give preflight response
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            "Access-Control-Request-Method": "POST",
+            "Access-Control-Request-Headers": "X-Custom",
+            })
+        eq_(r.status_code, 200)
+        if "access-control-allow-origin" not in r.headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-methods"], "GET, HEAD")
+        eq_(r.headers["access-control-allow-headers"], "X-Custom")
+
+    def test_post_bodies(self):
+        # Test JSON post bodies
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '{"hello": 1}')
+        eq_(r.status_code, 404) # wrong parameters
+
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '["hello"]')
+        eq_(r.status_code, 415) # not a dict
+
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '[hello]')
+        eq_(r.status_code, 400) # badly formatted JSON

tests/test_printf.py

@@ -6,7 +6,7 @@ from nose.tools import assert_raises
 from cStringIO import StringIO
 import sys
 
-from test_helpers import *
+from testutil.helpers import *
 
 class TestPrintf(object):
     def test_printf(self):
@@ -18,7 +18,7 @@ class TestPrintf(object):
             printf("hello, world: %d", 123)
             fprintf(test2, "hello too: %d", 123)
             test3 = sprintf("hello three: %d", 123)
-        except:
+        except Exception:
             sys.stdout = old_stdout
             raise
         sys.stdout = old_stdout

tests/test_rbtree.py

@@ -6,15 +6,15 @@ from nilmdb.utils.printf import *
 from nose.tools import *
 from nose.tools import assert_raises
 
-from nilmdb.rbtree import RBTree, RBNode
+from nilmdb.server.rbtree import RBTree, RBNode
 
-from test_helpers import *
+from testutil.helpers import *
 import unittest
 
 # set to False to skip live renders
 do_live_renders = False
 def render(tree, description = "", live = True):
-    import renderdot
+    import testutil.renderdot as renderdot
     r = renderdot.RBTreeRenderer(tree)
     return r.render(description, live and do_live_renders)