update test times

git-svn-id: https://bucket.mit.edu/svn/nilm/tags/nilmdb-orig-tests@11322 ddd99763-3ecb-0310-9145-efcb8ce7c51f

.coveragerc

@@ -1,10 +0,0 @@
-# -*- conf -*-
-
-[run]
-# branch = True
-
-[report]
-exclude_lines =
-	pragma: no cover
-	if 0:
-omit = nilmdb/utils/datetime_tz*,nilmdb/scripts,nilmdb/_version.py

.gitattributes

@@ -1 +0,0 @@
-nilmdb/_version.py export-subst

.gitignore

@@ -1,27 +0,0 @@
-# Tests
-tests/*testdb/
-.coverage
-db/
-
-# Compiled / cythonized files
-docs/*.html
-build/
-*.pyc
-nilmdb/server/interval.c
-nilmdb/server/interval.so
-nilmdb/server/layout.c
-nilmdb/server/layout.so
-nilmdb/server/rbtree.c
-nilmdb/server/rbtree.so
-
-# Setup junk
-dist/
-nilmdb.egg-info/
-
-# This gets generated as needed by setup.py
-MANIFEST.in
-MANIFEST
-
-# Misc
-timeit*out
-

.pylintrc

@@ -1,250 +0,0 @@
-# -*- conf -*-
-[MASTER]
-
-# Specify a configuration file.
-#rcfile=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Profiled execution.
-profile=no
-
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=datetime_tz
-
-# Pickle collected data for later comparisons.
-persistent=no
-
-# List of plugins (as comma separated values of python modules names) to load,
-# usually to register additional checkers.
-load-plugins=
-
-
-[MESSAGES CONTROL]
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time.
-#enable=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once).
-disable=C0111,R0903,R0201,R0914,R0912,W0142,W0703,W0702
-
-
-[REPORTS]
-
-# Set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html
-output-format=parseable
-
-# Include message's id in output
-include-ids=yes
-
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
-# Tells whether to display a full report or only the messages
-reports=yes
-
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-
-# Add a comment according to your evaluation note. This is used by the global
-# evaluation report (RP0004).
-comment=no
-
-
-[SIMILARITIES]
-
-# Minimum lines number of a similarity.
-min-similarity-lines=4
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-
-[TYPECHECK]
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set).
-ignored-classes=SQLObject
-
-# When zope mode is activated, add a predefined set of Zope acquired attributes
-# to generated-members.
-zope=no
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E0201 when accessed. Python regular
-# expressions are accepted.
-generated-members=REQUEST,acl_users,aq_parent
-
-
-[FORMAT]
-
-# Maximum number of characters on a single line.
-max-line-length=80
-
-# Maximum number of lines in a module
-max-module-lines=1000
-
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-indent-string='    '
-
-
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
-
-
-[VARIABLES]
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# A regular expression matching the beginning of the name of dummy variables
-# (i.e. not used).
-dummy-variables-rgx=_|dummy
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
-
-
-[BASIC]
-
-# Required attributes for module, separated by a comma
-required-attributes=
-
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=apply,input
-
-# Regular expression which should only match correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
-# Regular expression which should only match correct module level names
-const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|version)$
-
-# Regular expression which should only match correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
-
-# Regular expression which should only match correct function names
-function-rgx=[a-z_][a-z0-9_]{0,30}$
-
-# Regular expression which should only match correct method names
-method-rgx=[a-z_][a-z0-9_]{0,30}$
-
-# Regular expression which should only match correct instance attribute names
-attr-rgx=[a-z_][a-z0-9_]{0,30}$
-
-# Regular expression which should only match correct argument names
-argument-rgx=[a-z_][a-z0-9_]{0,30}$
-
-# Regular expression which should only match correct variable names
-variable-rgx=[a-z_][a-z0-9_]{0,30}$
-
-# Regular expression which should only match correct list comprehension /
-# generator expression variable names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_
-
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
-
-# Regular expression which should only match functions or classes name which do
-# not require a docstring
-no-docstring-rgx=__.*__
-
-
-[CLASSES]
-
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defines in Zope's Interface base class.
-ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
-
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-
-
-[DESIGN]
-
-# Maximum number of arguments for function / method
-max-args=5
-
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore
-ignored-argument-names=_.*
-
-# Maximum number of locals for function / method body
-max-locals=15
-
-# Maximum number of return / yield for function / method body
-max-returns=6
-
-# Maximum number of branch for function / method body
-max-branchs=12
-
-# Maximum number of statements in function / method body
-max-statements=50
-
-# Maximum number of parents for a class (see R0901).
-max-parents=7
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
-
-
-[IMPORTS]
-
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
-
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
-
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
-
-
-[EXCEPTIONS]
-
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception

Makefile

@@ -1,36 +1,2 @@
-# By default, run the tests.
-all: test
-
-version:
-	python setup.py version
-
-build:
-	python setup.py build_ext --inplace
-
-dist: sdist
-sdist:
-	python setup.py sdist
-
-install:
-	python setup.py install
-
-docs:
-	make -C docs
-
-lint:
-	pylint --rcfile=.pylintrc nilmdb
-
-test:
-	python tests/runtests.py
-
-clean::
-	find . -name '*pyc' | xargs rm -f
-	rm -f .coverage
-	rm -rf tests/*testdb*
-	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so MANIFEST.in
-	make -C docs clean
-
-gitclean::
-	git clean -dXf
-
-.PHONY: all build dist sdist install docs lint test clean
+all:
+	nosetests nilmdb/test_interval.py

README.txt

@@ -1,22 +1,4 @@
-nilmdb: Non-Intrusive Load Monitor Database
-by Jim Paris <jim@jtan.com>
+To install, 
 
-Prerequisites:
+   python seutp.py install
 
-  # Runtime and build environments
-  sudo apt-get install python2.7 python2.7-dev python-setuptools cython
-
-  # Base NilmDB dependencies
-  sudo apt-get install python-cherrypy3 python-decorator python-simplejson python-pycurl python-dateutil python-tz python-psutil
-
-  # Tools for running tests
-  sudo apt-get install python-nose python-coverage
-
-Install:
-
-  python setup.py install
-
-Usage:
-
-  nilmdb-server --help
-  nilmtool --help

bin/nilm-test.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+
+from nilmdb import Interval
+from optparse import OptionParser
+import sys
+
+version = "1.0"
+
+parser = OptionParser()
+parser.add_option("-d", "--db", dest="database", metavar="DATABASE",
+                  help="location of sqlite database")
+parser.add_option("-V", "--version", dest="version", default=False, action="store_true",
+                  help="print version then exit")
+
+(options, args) = parser.parse_args()
+
+if (options.version):
+    print "This script version: " + version
+    sys.exit(0)
+
+if options.database is None:
+    print "Error: database is mandatory"
+    sys.exit(1)
+
+print "Database is " + options.database
+

docs/Makefile

@@ -1,9 +0,0 @@
-ALL_DOCS = $(wildcard *.md)
-
-all: $(ALL_DOCS:.md=.html)
-
-%.html: %.md
-	pandoc -s $< > $@
-
-clean:
-	rm -f *.html

docs/TODO.md

@@ -1,5 +0,0 @@
-- Documentation
-
-- Machine-readable information in OverflowError, parser errors.
-  Maybe subclass `cherrypy.HTTPError` and override `set_response`
-  to add another JSON field?

docs/design.md

@@ -1,268 +0,0 @@
-Structure
----------
-nilmdb.nilmdb is the NILM database interface.  A nilmdb.BulkData
-interface stores data in flat files, and a SQL database tracks
-metadata and ranges.
-
-Access to the nilmdb must be single-threaded.  This is handled with
-the nilmdb.serializer class.  In the future this could probably
-be turned into a per-path serialization.
-
-nilmdb.server is a HTTP server that provides an interface to talk,
-thorugh the serialization layer, to the nilmdb object.
-
-nilmdb.client is a HTTP client that connects to this.
-
-Sqlite performance
-------------------
-
-Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
-takes about 125msec.  sqlite3 will commit transactions at 3 times:
-
-1. explicit con.commit()
-
-2. between a series of DML commands and non-DML commands, e.g.
-   after a series of INSERT, SELECT, but before a CREATE TABLE or
-   PRAGMA.
-
-3. at the end of an explicit transaction, e.g. "with self.con as con:"
-
-To speed up testing, or if this transaction speed becomes an issue,
-the sync=False option to NilmDB will set PRAGMA synchronous=OFF.
-
-
-Inserting streams
------------------
-
-We need to send the contents of "data" as POST.  Do we need chunked
-transfer?
-
-- Don't know the size in advance, so we would need to use chunked if
-  we send the entire thing in one request.
-- But we shouldn't send one chunk per line, so we need to buffer some
-  anyway; why not just make new requests?
-- Consider the infinite-streaming case, we might want to send it
-  immediately?  Not really -- server still should do explicit inserts
-  of fixed-size chunks.
-- Even chunked encoding needs the size of each chunk beforehand, so
-  everything still gets buffered.  Just a tradeoff of buffer size.
-
-Before timestamps are added:
-
-- Raw data is about 440 kB/s    (9 channels)
-- Prep data is about 12.5 kB/s  (1 phase)
-- How do we know how much data to send?
-
-    - Remember that we can only do maybe 8-50 transactions per second on
-      the sqlite database.  So if one block of inserted data is one
-      transaction, we'd need the raw case to be around 64kB per request,
-      ideally more.
-    - Maybe use a range, based on how long it's taking to read the data
-        - If no more data, send it
-        - If data > 1 MB, send it
-    - If more than 10 seconds have elapsed, send it
-    - Should those numbers come from the server?
-
-Converting from ASCII to PyTables:
-
-- For each row getting added, we need to set attributes on a PyTables
-  Row object and call table.append().  This means that there isn't a
-  particularly efficient way of converting from ascii.
-- Could create a function like nilmdb.layout.Layout("foo".fillRow(asciiline)
-    - But this means we're doing parsing on the serialized side
-    - Let's keep parsing on the threaded server side so we can detect
-      errors better, and not block the serialized nilmdb for a slow
-      parsing process.
-- Client sends ASCII data
-- Server converts this ACSII data to a list of values
-    - Maybe:
-
-            # threaded side creates this object
-            parser = nilmdb.layout.Parser("layout_name")
-            # threaded side parses and fills it with data
-            parser.parse(textdata)
-            # serialized side pulls out rows
-            for n in xrange(parser.nrows):
-                parser.fill_row(rowinstance, n)
-                table.append()
-
-
-Inserting streams, inside nilmdb
---------------------------------
-
-- First check that the new stream doesn't overlap.
-    - Get minimum timestamp, maximum timestamp from data parser.
-        - (extend parser to verify monotonicity and track extents)
-    - Get all intervals for this stream in the database
-    - See if new interval overlaps any existing ones
-        - If so, bail
-    - Question: should we cache intervals inside NilmDB?
-        - Assume database is fast for now, and always rebuild fom DB.
-        - Can add a caching layer later if we need to.
-    - `stream_get_ranges(path)` -> return IntervalSet?
-
-Speed
------
-
-- First approach was quadratic.  Adding four hours of data:
-
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/1/raw
-        real    24m31.093s
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/1/raw
-        real    43m44.528s
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-130002 /bpnilm/1/raw
-        real    93m29.713s
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-140003 /bpnilm/1/raw
-        real    166m53.007s
-
-- Disabling pytables indexing didn't help:
-
-        real    31m21.492s
-        real    52m51.963s
-        real    102m8.151s
-        real    176m12.469s
-
-- Server RAM usage is constant.
-
-- Speed problems were due to IntervalSet speed, of parsing intervals
-  from the database and adding the new one each time.
-
-    - First optimization is to cache result of `nilmdb:_get_intervals`,
-      which gives the best speedup.
-
-    - Also switched to internally using bxInterval from bx-python package.
-      Speed of `tests/test_interval:TestIntervalSpeed` is pretty decent
-      and seems to be growing logarithmically now.  About 85μs per insertion
-      for inserting 131k entries.
-
-    - Storing the interval data in SQL might be better, with a scheme like:
-      http://www.logarithmic.net/pfh/blog/01235197474
-
-- Next slowdown target is nilmdb.layout.Parser.parse().
-    - Rewrote parsers using cython and sscanf
-    - Stats (rev 10831), with _add_interval disabled
-
-        layout.pyx.Parser.parse:128        6303 sec, 262k calls
-         layout.pyx.parse:63               13913 sec, 5.1g calls
-        numpy:records.py.fromrecords:569   7410 sec, 262k calls
-
-- Probably OK for now.
-
-- After all updates, now takes about 8.5 minutes to insert an hour of
-  data, constant after adding 171 hours (4.9 billion data points)
-
-- Data set size: 98 gigs = 20 bytes per data point.
-  6 uint16 data + 1 uint32 timestamp = 16 bytes per point
-  So compression must be off -- will retry with compression forced on.
-
-IntervalSet speed
------------------
-- Initial implementation was pretty slow, even with binary search in
-  sorted list
-
-- Replaced with bxInterval; now takes about log n time for an insertion
-    - TestIntervalSpeed with range(17,18) and profiling
-        - 85 μs each
-        - 131072 calls to `__iadd__`
-        - 131072 to bx.insert_interval
-        - 131072 to bx.insert:395
-        - 2355835 to bx.insert:106  (18x as many?)
-
-- Tried blist too, worse than bxinterval.
-
-- Might be algorithmic improvements to be made in Interval.py,
-  like in `__and__`
-
-- Replaced again with rbtree.  Seems decent.  Numbers are time per
-  insert for 2**17 insertions, followed by total wall time and RAM
-  usage for running "make test" with `test_rbtree` and `test_interval`
-  with range(5,20):
-    - old values with bxinterval:
-      20.2 μS, total 20 s, 177 MB RAM
-    - rbtree, plain python:
-      97 μS, total 105 s, 846 MB RAM
-    - rbtree converted to cython:
-      26 μS, total 29 s, 320 MB RAM
-    - rbtree and interval converted to cython:
-      8.4 μS, total 12 s, 134 MB RAM
-
-Layouts
--------
-Current/old design has specific layouts: RawData, PrepData, RawNotchedData.
-Let's get rid of this entirely and switch to simpler data types that are
-just collections and counts of a single type.  We'll still use strings
-to describe them, with format:
-
-    type_count
-
-where type is "uint16", "float32", or "float64", and count is an integer.
-
-nilmdb.layout.named() will parse these strings into the appropriate
-handlers.  For compatibility:
-
-    "RawData" == "uint16_6"
-    "RawNotchedData" == "uint16_9"
-    "PrepData" == "float32_8"
-
-
-BulkData design
----------------
-
-BulkData is a custom bulk data storage system that was written to
-replace PyTables.  The general structure is a `data` subdirectory in
-the main NilmDB directory.  Within `data`, paths are created for each
-created stream.  These locations are called tables.  For example,
-tables might be located at
-
-    nilmdb/data/newton/raw/
-    nilmdb/data/newton/prep/
-    nilmdb/data/cottage/raw/
-
-Each table contains:
-
-- An unchanging `_format` file (Python pickle format) that describes
-  parameters of how the data is broken up, like files per directory,
-  rows per file, and the binary data format
-
-- Hex named subdirectories `("%04x", although more than 65536 can exist)`
-
-- Hex named files within those subdirectories, like:
-
-        /nilmdb/data/newton/raw/000b/010a
-
-    The data format of these files is raw binary, interpreted by the
-    Python `struct` module according to the format string in the
-    `_format` file.
-
-- Same as above, with `.removed` suffix, is an optional file (Python
-  pickle format) containing a list of row numbers that have been
-  logically removed from the file.  If this range covers the entire
-  file, the entire file will be removed.
-
-- Note that the `bulkdata.nrows` variable is calculated once in
-  `BulkData.__init__()`, and only ever incremented during use.  Thus,
-  even if all data is removed, `nrows` can remain high.  However, if
-  the server is restarted, the newly calculated `nrows` may be lower
-  than in a previous run due to deleted data.  To be specific, this
-  sequence of events:
-
-    - insert data
-    - remove all data
-    - insert data
-
-    will result in having different row numbers in the database, and
-    differently numbered files on the filesystem, than the sequence:
-
-    - insert data
-    - remove all data
-    - restart server
-    - insert data
-
-    This is okay!  Everything should remain consistent both in the
-    `BulkData` and `NilmDB`.  Not attempting to readjust `nrows` during
-    deletion makes the code quite a bit simpler.
-
-- Similarly, data files are never truncated shorter.  Removing data
-  from the end of the file will not shorten it; it will only be
-  deleted when it has been fully filled and all of the data has been
-  subsequently removed.

nilmdb/__init__.py

@@ -1,8 +1,2 @@
-"""Main NilmDB import"""
-
-from nilmdb.server import NilmDB, Server
-from nilmdb.client import Client
-
-from nilmdb._version import get_versions
-__version__ = get_versions()['version']
-del get_versions
+from nilmdb.interval import *
+from nilmdb.fileinterval import *

nilmdb/_version.py

@@ -1,197 +0,0 @@
-
-IN_LONG_VERSION_PY = True
-# This file helps to compute a version number in source trees obtained from
-# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
-# directories (produced by setup.py build) will contain a much shorter file
-# that just contains the computed version number.
-
-# This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
-
-# these strings will be replaced by git during git-archive
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
-
-
-import subprocess
-import sys
-
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.cmd on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
-        if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
-        return None
-    stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % args[0])
-        return None
-    return stdout
-
-
-import sys
-import re
-import os.path
-
-def get_expanded_variables(versionfile_source):
-    # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
-    try:
-        for line in open(versionfile_source,"r").readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    variables["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    variables["full"] = mo.group(1)
-    except EnvironmentError:
-        pass
-    return variables
-
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
-    if refnames.startswith("$Format"):
-        if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
-    for ref in list(refs):
-        if not re.search(r'\d', ref):
-            if verbose:
-                print("discarding '%s', no digits" % ref)
-            refs.discard(ref)
-            # Assume all version tags have a digit. git's %d expansion
-            # behaves like git log --decorate=short and strips out the
-            # refs/heads/ and refs/tags/ prefixes that would let us
-            # distinguish between branches and tags. By ignoring refnames
-            # without digits, we filter out many common branch names like
-            # "release" and "stabilization", as well as "HEAD" and "master".
-    if verbose:
-        print("remaining refs: %s" % ",".join(sorted(refs)))
-    for ref in sorted(refs):
-        # sorting will prefer e.g. "2.0" over "2.0rc1"
-        if ref.startswith(tag_prefix):
-            r = ref[len(tag_prefix):]
-            if verbose:
-                print("picking %s" % r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
-    if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
-
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
-    if not os.path.exists(os.path.join(root, ".git")):
-        if verbose:
-            print("no .git in %s" % root)
-        return {}
-
-    GIT = "git"
-    if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
-
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = "nilmdb-"
-parentdir_prefix = "nilmdb-"
-versionfile_source = "nilmdb/_version.py"
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = { "refnames": git_refnames, "full": git_full }
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
-

nilmdb/client/__init__.py

@@ -1,4 +0,0 @@
-"""nilmdb.client"""
-
-from nilmdb.client.client import Client
-from nilmdb.client.errors import ClientError, ServerError, Error

nilmdb/client/client.py

@@ -1,379 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Class for performing HTTP client requests via libcurl"""
-
-import nilmdb
-import nilmdb.utils
-import nilmdb.client.httpclient
-
-import time
-import simplejson as json
-import contextlib
-
-def float_to_string(f):
-    """Use repr to maintain full precision in the string output."""
-    return repr(float(f))
-
-def extract_timestamp(line):
-    """Extract just the timestamp from a line of data text"""
-    return float(line.split()[0])
-
-class Client(object):
-    """Main client interface to the Nilm database."""
-
-    def __init__(self, url):
-        self.http = nilmdb.client.httpclient.HTTPClient(url)
-
-    # __enter__/__exit__ allow this class to be a context manager
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-
-    def _json_param(self, data):
-        """Return compact json-encoded version of parameter"""
-        return json.dumps(data, separators=(',',':'))
-
-    def close(self):
-        """Close the connection; safe to call multiple times"""
-        self.http.close()
-
-    def geturl(self):
-        """Return the URL we're using"""
-        return self.http.baseurl
-
-    def version(self):
-        """Return server version"""
-        return self.http.get("version")
-
-    def dbinfo(self):
-        """Return server database info (path, size, free space)
-        as a dictionary."""
-        return self.http.get("dbinfo")
-
-    def stream_list(self, path = None, layout = None):
-        params = {}
-        if path is not None:
-            params["path"] = path
-        if layout is not None:
-            params["layout"] = layout
-        return self.http.get("stream/list", params)
-
-    def stream_get_metadata(self, path, keys = None):
-        params = { "path": path }
-        if keys is not None:
-            params["key"] = keys
-        return self.http.get("stream/get_metadata", params)
-
-    def stream_set_metadata(self, path, data):
-        """Set stream metadata from a dictionary, replacing all existing
-        metadata."""
-        params = {
-            "path": path,
-            "data": self._json_param(data)
-            }
-        return self.http.get("stream/set_metadata", params)
-
-    def stream_update_metadata(self, path, data):
-        """Update stream metadata from a dictionary"""
-        params = {
-            "path": path,
-            "data": self._json_param(data)
-            }
-        return self.http.get("stream/update_metadata", params)
-
-    def stream_create(self, path, layout):
-        """Create a new stream"""
-        params = { "path": path,
-                   "layout" : layout }
-        return self.http.get("stream/create", params)
-
-    def stream_destroy(self, path):
-        """Delete stream and its contents"""
-        params = { "path": path }
-        return self.http.get("stream/destroy", params)
-
-    def stream_remove(self, path, start = None, end = None):
-        """Remove data from the specified time range"""
-        params = {
-            "path": path
-        }
-        if start is not None:
-            params["start"] = float_to_string(start)
-        if end is not None:
-            params["end"] = float_to_string(end)
-        return self.http.get("stream/remove", params)
-
-    @contextlib.contextmanager
-    def stream_insert_context(self, path, start = None, end = None):
-        """Return a context manager that allows data to be efficiently
-        inserted into a stream in a piecewise manner.  Data is be provided
-        as single lines, and is aggregated and sent to the server in larger
-        chunks as necessary.  Data lines must match the database layout for
-        the given path, and end with a newline.
-
-        Example:
-          with client.stream_insert_context('/path', start, end) as ctx:
-            ctx.insert_line('1234567890.0 1 2 3 4\\n')
-            ctx.insert_line('1234567891.0 1 2 3 4\\n')
-
-        For more details, see help for nilmdb.client.client.StreamInserter
-
-        This may make multiple requests to the server, if the data is
-        large enough or enough time has passed between insertions.
-        """
-        ctx = StreamInserter(self, path, start, end)
-        yield ctx
-        ctx.finalize()
-
-    def stream_insert(self, path, data, start = None, end = None):
-        """Insert rows of data into a stream.  data should be an
-        iterable object that provides ASCII data that matches the
-        database layout for path.  See stream_insert_context for
-        details on the 'start' and 'end' parameters."""
-        with self.stream_insert_context(path, start, end) as ctx:
-            ctx.insert_iter(data)
-        return ctx.last_response
-
-    def stream_insert_block(self, path, block, start, end):
-        """Insert an entire block of data into a stream.  Like
-        stream_insert, except 'block' contains multiple lines of ASCII
-        text and is sent in one single chunk."""
-        params = { "path": path,
-                   "start": float_to_string(start),
-                   "end": float_to_string(end) }
-        return self.http.put("stream/insert", block, params)
-
-    def stream_intervals(self, path, start = None, end = None):
-        """
-        Return a generator that yields each stream interval.
-        """
-        params = {
-            "path": path
-        }
-        if start is not None:
-            params["start"] = float_to_string(start)
-        if end is not None:
-            params["end"] = float_to_string(end)
-        return self.http.get_gen("stream/intervals", params, retjson = True)
-
-    def stream_extract(self, path, start = None, end = None, count = False):
-        """
-        Extract data from a stream.  Returns a generator that yields
-        lines of ASCII-formatted data that matches the database
-        layout for the given path.
-
-        Specify count = True to return a count of matching data points
-        rather than the actual data.  The output format is unchanged.
-        """
-        params = {
-            "path": path,
-        }
-        if start is not None:
-            params["start"] = float_to_string(start)
-        if end is not None:
-            params["end"] = float_to_string(end)
-        if count:
-            params["count"] = 1
-
-        return self.http.get_gen("stream/extract", params, retjson = False)
-
-    def stream_count(self, path, start = None, end = None):
-        """
-        Return the number of rows of data in the stream that satisfy
-        the given timestamps.
-        """
-        counts = list(self.stream_extract(path, start, end, count = True))
-        return int(counts[0])
-
-class StreamInserter(object):
-    """Object returned by stream_insert_context() that manages
-    the insertion of rows of data into a particular path.
-
-    The basic data flow is that we are filling a contiguous interval
-    on the server, with no gaps, that extends from timestamp 'start'
-    to timestamp 'end'.  Data timestamps satisfy 'start <= t < end'.
-    Data is provided by the user one line at a time with
-    .insert_line() or .insert_iter().
-
-    1. The first inserted line begins a new interval that starts at
-    'start'.  If 'start' is not given, it is deduced from the first
-    line's timestamp.
-
-    2. Subsequent lines go into the same contiguous interval.  As lines
-    are inserted, this routine may make multiple insertion requests to
-    the server, but will structure the timestamps to leave no gaps.
-
-    3. The current contiguous interval can be completed by manually
-    calling .finalize(), which the context manager will also do
-    automatically.  This will send any remaining data to the server,
-    using the 'end' timestamp to end the interval.
-
-    After a .finalize(), inserting new data goes back to step 1.
-
-    .update_start() can be called before step 1 to change the start
-    time for the interval.  .update_end() can be called before step 3
-    to change the end time for the interval.
-    """
-
-    # See design.md for a discussion of how much data to send.
-    # These are soft limits -- actual data might be rounded up.
-    # We send when we have a certain amount of data queued, or
-    # when a certain amount of time has passed since the last send.
-    _max_data = 1048576
-    _max_time = 30
-
-    # Delta to add to the final timestamp, if "end" wasn't given
-    _end_epsilon = 1e-6
-
-    def __init__(self, client, path, start = None, end = None):
-        """'http' is the httpclient object.  'path' is the database
-        path to insert to.  'start' and 'end' are used for the first
-        contiguous interval."""
-        self.last_response = None
-
-        self._client = client
-        self._path = path
-
-        # Start and end for the overall contiguous interval we're
-        # filling
-        self._interval_start = start
-        self._interval_end = end
-
-        # Data for the specific block we're building up to send
-        self._block_data = []
-        self._block_len = 0
-        self._block_start = None
-
-        # Time of last request
-        self._last_time = time.time()
-
-        # We keep a buffer of the two most recently inserted lines.
-        # Only the older one actually gets processed; the newer one
-        # is used to "look-ahead" to the next timestamp if we need
-        # to internally split an insertion into two requests.
-        self._line_old = None
-        self._line_new = None
-
-    def insert_iter(self, iter):
-        """Insert all lines of ASCII formatted data from the given
-        iterable.  Lines must be terminated with '\\n'."""
-        for line in iter:
-            self.insert_line(line)
-
-    def insert_line(self, line, allow_intermediate = True):
-        """Insert a single line of ASCII formatted data.  Line
-        must be terminated with '\\n'."""
-        if line and (len(line) < 1 or line[-1] != '\n'):
-            raise ValueError("lines must end in with a newline character")
-
-        # Store this new line, but process the previous (old) one.
-        # This lets us "look ahead" to the next line.
-        self._line_old = self._line_new
-        self._line_new = line
-        if self._line_old is None:
-            return
-
-        # If starting a new block, pull out the timestamp if needed.
-        if self._block_start is None:
-            if self._interval_start is not None:
-                # User provided a start timestamp.  Use it once, then
-                # clear it for the next block.
-                self._block_start = self._interval_start
-                self._interval_start = None
-            else:
-                # Extract timestamp from the first row
-                self._block_start = extract_timestamp(self._line_old)
-
-        # Save the line
-        self._block_data.append(self._line_old)
-        self._block_len += len(self._line_old)
-
-        if allow_intermediate:
-            # Send an intermediate block to the server if needed.
-            elapsed = time.time() - self._last_time
-            if (self._block_len > self._max_data) or (elapsed > self._max_time):
-                self._send_block_intermediate()
-
-    def update_start(self, start):
-        """Update the start time for the next contiguous interval.
-        Call this before starting to insert data for a new interval,
-        for example, after .finalize()"""
-        self._interval_start = start
-
-    def update_end(self, end):
-        """Update the end time for the current contiguous interval.
-        Call this before .finalize()"""
-        self._interval_end = end
-
-    def finalize(self):
-        """Stop filling the current contiguous interval.
-        All outstanding data will be sent, and the interval end
-        time of the interval will be taken from the 'end' argument
-        used when initializing this class, or the most recent
-        value passed to update_end(), or the last timestamp plus
-        a small epsilon value if no other endpoint was provided.
-
-        If more data is inserted after a finalize(), it will become
-        part of a new interval and there may be a gap left in-between."""
-        # Special marker tells insert_line that this is the end
-        self.insert_line(None, allow_intermediate = False)
-
-        if self._block_len > 0:
-            # We have data pending, so send the final block
-            self._send_block_final()
-        elif None not in (self._interval_start, self._interval_end):
-            # We have no data, but enough information to create an
-            # empty interval.
-            self._block_start = self._interval_start
-            self._interval_start = None
-            self._send_block_final()
-        else:
-            # No data, and no timestamps to use to create an empty
-            # interval.
-            pass
-
-        # Make sure both timestamps are emptied for future intervals.
-        self._interval_start = None
-        self._interval_end = None
-
-    def _send_block_intermediate(self):
-        """Send data, when we still have more data to send.
-        Use the timestamp from the next line, so that the blocks
-        are contiguous."""
-        block_end = extract_timestamp(self._line_new)
-        if self._interval_end is not None and block_end > self._interval_end:
-            # Something's fishy -- the timestamp we found is after
-            # the user's specified end.  Limit it here, and the
-            # server will return an error.
-            block_end = self._interval_end
-        self._send_block(block_end)
-
-    def _send_block_final(self):
-        """Send data, when this is the last block for the interval.
-        There is no next line, so figure out the actual interval end
-        using interval_end or end_epsilon."""
-        if self._interval_end is not None:
-            # Use the user's specified end timestamp
-            block_end = self._interval_end
-            # Clear it in case we send more intervals in the future.
-            self._interval_end = None
-        else:
-            # Add an epsilon to the last timestamp we saw
-            block_end = extract_timestamp(self._line_old) + self._end_epsilon
-        self._send_block(block_end)
-
-    def _send_block(self, block_end):
-        """Send current block to the server"""
-        self.last_response = self._client.stream_insert_block(
-            self._path, "".join(self._block_data),
-            self._block_start, block_end)
-
-        # Clear out the block
-        self._block_data = []
-        self._block_len = 0
-        self._block_start = None
-
-        # Note when we sent it
-        self._last_time = time.time()

nilmdb/client/errors.py

@@ -1,33 +0,0 @@
-"""HTTP client errors"""
-
-from nilmdb.utils.printf import *
-
-class Error(Exception):
-    """Base exception for both ClientError and ServerError responses"""
-    def __init__(self,
-                 status = "Unspecified error",
-                 message = None,
-                 url = None,
-                 traceback = None):
-        Exception.__init__(self, status)
-        self.status = status     # e.g. "400 Bad Request"
-        self.message = message   # textual message from the server
-        self.url = url           # URL we were requesting
-        self.traceback = traceback # server traceback, if available
-    def _format_error(self, show_url):
-        s = sprintf("[%s]", self.status)
-        if self.message:
-            s += sprintf(" %s", self.message)
-        if show_url and self.url: # pragma: no cover
-            s += sprintf(" (%s)", self.url)
-        if self.traceback: # pragma: no cover
-            s += sprintf("\nServer traceback:\n%s", self.traceback)
-        return s
-    def __str__(self):
-        return self._format_error(show_url = False)
-    def __repr__(self): # pragma: no cover
-        return self._format_error(show_url = True)
-class ClientError(Error):
-    pass
-class ServerError(Error):
-    pass

nilmdb/client/httpclient.py

@@ -1,208 +0,0 @@
-"""HTTP client library"""
-
-import nilmdb
-import nilmdb.utils
-from nilmdb.client.errors import ClientError, ServerError, Error
-
-import simplejson as json
-import urlparse
-import pycurl
-import cStringIO
-
-class HTTPClient(object):
-    """Class to manage and perform HTTP requests from the client"""
-    def __init__(self, baseurl = ""):
-        """If baseurl is supplied, all other functions that take
-        a URL can be given a relative URL instead."""
-        # Verify / clean up URL
-        reparsed = urlparse.urlparse(baseurl).geturl()
-        if '://' not in reparsed:
-            reparsed = urlparse.urlparse("http://" + baseurl).geturl()
-        self.baseurl = reparsed
-        self.curl = pycurl.Curl()
-        self.curl.setopt(pycurl.SSL_VERIFYHOST, 2)
-        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
-        self.curl.setopt(pycurl.MAXREDIRS, 5)
-        self._setup_url()
-
-    def _setup_url(self, url = "", params = ""):
-        url = urlparse.urljoin(self.baseurl, url)
-        if params:
-            url = urlparse.urljoin(
-                url, "?" + nilmdb.utils.urllib.urlencode(params))
-        self.curl.setopt(pycurl.URL, url)
-        self.url = url
-
-    def _check_busy_and_set_upload(self, upload):
-        """Sets the pycurl.UPLOAD option, but also raises a more
-        friendly exception if the client is already serving a request."""
-        try:
-            self.curl.setopt(pycurl.UPLOAD, upload)
-        except pycurl.error as e:
-            if "is currently running" in str(e):
-                raise Exception("Client is already performing a request, and "
-                                "nesting calls is not supported.")
-            else: # pragma: no cover (shouldn't happen)
-                raise
-
-    def _check_error(self, body = None):
-        code = self.curl.getinfo(pycurl.RESPONSE_CODE)
-        if code == 200:
-            return
-        # Default variables for exception.  We use the entire body as
-        # the default message, in case we can't extract it from a JSON
-        # response.
-        args = { "url" : self.url,
-                 "status" : str(code),
-                 "message" : body,
-                 "traceback" : None }
-        try:
-            # Fill with server-provided data if we can
-            jsonerror = json.loads(body)
-            args["status"] = jsonerror["status"]
-            args["message"] = jsonerror["message"]
-            args["traceback"] = jsonerror["traceback"]
-        except Exception: # pragma: no cover
-            pass
-        if code >= 400 and code <= 499:
-            raise ClientError(**args)
-        else: # pragma: no cover
-            if code >= 500 and code <= 599:
-                if args["message"] is None:
-                    args["message"] = ("(no message; try disabling " +
-                                       "response.stream option in " +
-                                       "nilmdb.server for better debugging)")
-                raise ServerError(**args)
-            else:
-                raise Error(**args)
-
-    def _req_generator(self, url, params):
-        """
-        Like self._req(), but runs the perform in a separate thread.
-        It returns a generator that spits out arbitrary-sized chunks
-        of the resulting data, instead of using the WRITEFUNCTION
-        callback.
-        """
-        self._setup_url(url, params)
-        self._status = None
-        error_body = ""
-        self._headers = ""
-        def header_callback(data):
-            if self._status is None:
-                self._status = int(data.split(" ")[1])
-            self._headers += data
-        self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
-        def perform(callback):
-            self.curl.setopt(pycurl.WRITEFUNCTION, callback)
-            self.curl.perform()
-        try:
-            with nilmdb.utils.Iteratorizer(perform, curl_hack = True) as it:
-                for i in it:
-                    if self._status == 200:
-                        # If we had a 200 response, yield the data to caller.
-                        yield i
-                    else:
-                        # Otherwise, collect it into an error string.
-                        error_body += i
-        except pycurl.error as e:
-            raise ServerError(status = "502 Error",
-                              url = self.url,
-                              message = e[1])
-        # Raise an exception if there was an error
-        self._check_error(error_body)
-
-    def _req(self, url, params):
-        """
-        GET or POST that returns raw data.  Returns the body
-        data as a string, or raises an error if it contained an error.
-        """
-        self._setup_url(url, params)
-        body = cStringIO.StringIO()
-        self.curl.setopt(pycurl.WRITEFUNCTION, body.write)
-        self._headers = ""
-        def header_callback(data):
-            self._headers += data
-        self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
-        try:
-            self.curl.perform()
-        except pycurl.error as e:
-            raise ServerError(status = "502 Error",
-                              url = self.url,
-                              message = e[1])
-        body_str = body.getvalue()
-        # Raise an exception if there was an error
-        self._check_error(body_str)
-        return body_str
-
-    def close(self):
-        self.curl.close()
-
-    def _iterate_lines(self, it):
-        """
-        Given an iterator that returns arbitrarily-sized chunks
-        of data, return '\n'-delimited lines of text
-        """
-        partial = ""
-        for chunk in it:
-            partial += chunk
-            lines = partial.split("\n")
-            for line in lines[0:-1]:
-                yield line
-            partial = lines[-1]
-        if partial != "":
-            yield partial
-
-    # Non-generator versions
-    def _doreq(self, url, params, retjson):
-        """
-        Perform a request, and return the body.
-
-        url: URL to request (relative to baseurl)
-        params: dictionary of query parameters
-        retjson: expect JSON and return python objects instead of string
-        """
-        out = self._req(url, params)
-        if retjson:
-            return json.loads(out)
-        return out
-
-    def get(self, url, params = None, retjson = True):
-        """Simple GET"""
-        self._check_busy_and_set_upload(0)
-        return self._doreq(url, params, retjson)
-
-    def put(self, url, postdata, params = None, retjson = True):
-        """Simple PUT"""
-        self._check_busy_and_set_upload(1)
-        self._setup_url(url, params)
-        data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.READFUNCTION, data.read)
-        return self._doreq(url, params, retjson)
-
-    # Generator versions
-    def _doreq_gen(self, url, params, retjson):
-        """
-        Perform a request, and return lines of the body in a generator.
-
-        url: URL to request (relative to baseurl)
-        params: dictionary of query parameters
-        retjson: expect JSON and yield python objects instead of strings
-        """
-        for line in self._iterate_lines(self._req_generator(url, params)):
-            if retjson:
-                yield json.loads(line)
-            else:
-                yield line
-
-    def get_gen(self, url, params = None, retjson = True):
-        """Simple GET, returning a generator"""
-        self._check_busy_and_set_upload(0)
-        return self._doreq_gen(url, params, retjson)
-
-    def put_gen(self, url, postdata, params = None, retjson = True):
-        """Simple PUT, returning a generator"""
-        self._check_busy_and_set_upload(1)
-        self._setup_url(url, params)
-        data = cStringIO.StringIO(postdata)
-        self.curl.setopt(pycurl.READFUNCTION, data.read)
-        return self._doreq_gen(url, params, retjson)

nilmdb/cmdline/__init__.py

@@ -1,3 +0,0 @@
-"""nilmdb.cmdline"""
-
-from nilmdb.cmdline.cmdline import Cmdline

nilmdb/cmdline/cmdline.py

@@ -1,151 +0,0 @@
-"""Command line client functionality"""
-
-import nilmdb
-from nilmdb.utils.printf import *
-from nilmdb.utils import datetime_tz
-
-import sys
-import re
-import argparse
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-# Valid subcommands.  Defined in separate files just to break
-# things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
-                "remove", "destroy" ]
-
-# Import the subcommand modules
-subcmd_mods = {}
-for cmd in subcommands:
-    subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist = [ cmd ])
-
-class JimArgumentParser(argparse.ArgumentParser):
-    def error(self, message):
-        self.print_usage(sys.stderr)
-        self.exit(2, sprintf("error: %s\n", message))
-
-class Cmdline(object):
-
-    def __init__(self, argv = None):
-        self.argv = argv or sys.argv[1:]
-        self.client = None
-
-    def arg_time(self, toparse):
-        """Parse a time string argument"""
-        try:
-            return self.parse_time(toparse).totimestamp()
-        except ValueError as e:
-            raise argparse.ArgumentTypeError(sprintf("%s \"%s\"",
-                                                     str(e), toparse))
-
-    def parse_time(self, toparse):
-        """
-        Parse a free-form time string and return a datetime_tz object.
-        If the string doesn't contain a timestamp, the current local
-        timezone is assumed (e.g. from the TZ env var).
-        """
-        # If string isn't "now" and doesn't contain at least 4 digits,
-        # consider it invalid.  smartparse might otherwise accept
-        # empty strings and strings with just separators.
-        if toparse != "now" and len(re.findall(r"\d", toparse)) < 4:
-            raise ValueError("not enough digits for a timestamp")
-
-        # Try to just parse the time as given
-        try:
-            return datetime_tz.datetime_tz.smartparse(toparse)
-        except ValueError:
-            pass
-
-        # Try to extract a substring in a condensed format that we expect
-        # to see in a filename or header comment
-        res = re.search(r"(^|[^\d])("            # non-numeric or SOL
-                        r"(199\d|2\d\d\d)"       # year
-                        r"[-/]?"                 # separator
-                        r"(0[1-9]|1[012])"       # month
-                        r"[-/]?"                 # separator
-                        r"([012]\d|3[01])"       # day
-                        r"[-T ]?"                # separator
-                        r"([01]\d|2[0-3])"       # hour
-                        r"[:]?"                  # separator
-                        r"([0-5]\d)"             # minute
-                        r"[:]?"                  # separator
-                        r"([0-5]\d)?"            # second
-                        r"([-+]\d\d\d\d)?"       # timezone
-                        r")", toparse)
-        if res is not None:
-            try:
-                return datetime_tz.datetime_tz.smartparse(res.group(2))
-            except ValueError:
-                pass
-
-        # Could also try to successively parse substrings, but let's
-        # just give up for now.
-        raise ValueError("unable to parse timestamp")
-
-    def time_string(self, timestamp):
-        """
-        Convert a Unix timestamp to a string for printing, using the
-        local timezone for display (e.g. from the TZ env var).
-        """
-        dt = datetime_tz.datetime_tz.fromtimestamp(timestamp)
-        return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
-
-    def parser_setup(self):
-        self.parser = JimArgumentParser(add_help = False,
-                                        formatter_class = def_form)
-
-        group = self.parser.add_argument_group("General options")
-        group.add_argument("-h", "--help", action='help',
-                           help='show this help message and exit')
-        group.add_argument("-V", "--version", action="version",
-                           version = nilmdb.__version__)
-
-        group = self.parser.add_argument_group("Server")
-        group.add_argument("-u", "--url", action="store",
-                           default="http://localhost:12380/",
-                           help="NilmDB server URL (default: %(default)s)")
-
-        sub = self.parser.add_subparsers(title="Commands",
-                                         dest="command",
-                                         description="Specify --help after "
-                                         "the command for command-specific "
-                                         "options.")
-
-        # Set up subcommands (defined in separate files)
-        for cmd in subcommands:
-            subcmd_mods[cmd].setup(self, sub)
-
-    def die(self, formatstr, *args):
-        fprintf(sys.stderr, formatstr + "\n", *args)
-        if self.client:
-            self.client.close()
-        sys.exit(-1)
-
-    def run(self):
-        # Clear cached timezone, so that we can pick up timezone changes
-        # while running this from the test suite.
-        datetime_tz._localtz = None
-
-        # Run parser
-        self.parser_setup()
-        self.args = self.parser.parse_args(self.argv)
-
-        # Run arg verify handler if there is one
-        if "verify" in self.args:
-            self.args.verify(self)
-
-        self.client = nilmdb.Client(self.args.url)
-
-        # Make a test connection to make sure things work
-        try:
-            server_version = self.client.version()
-        except nilmdb.client.Error as e:
-            self.die("error connecting to server: %s", str(e))
-
-        # Now dispatch client request to appropriate function.  Parser
-        # should have ensured that we don't have any unknown commands
-        # here.
-        retval = self.args.handler(self) or 0
-
-        self.client.close()
-        sys.exit(retval)

nilmdb/cmdline/create.py

@@ -1,35 +0,0 @@
-from nilmdb.utils.printf import *
-import nilmdb
-import nilmdb.client
-
-from argparse import RawDescriptionHelpFormatter as raw_form
-
-def setup(self, sub):
-    cmd = sub.add_parser("create", help="Create a new stream",
-                         formatter_class = raw_form,
-                         description="""
-Create a new empty stream at the specified path and with the specified
-layout type.
-
-Layout types are of the format: type_count
-
-  'type' is a data type like 'float32', 'float64', 'uint16', 'int32', etc.
-
-  'count' is the number of columns of this type.
-
-  For example, 'float32_8' means the data for this stream has 8 columns of
-  32-bit floating point values.
-""")
-    cmd.set_defaults(handler = cmd_create)
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("path",
-                       help="Path (in database) of new stream, e.g. /foo/bar")
-    group.add_argument("layout",
-                       help="Layout type for new stream, e.g. float32_8")
-
-def cmd_create(self):
-    """Create new stream"""
-    try:
-        self.client.stream_create(self.args.path, self.args.layout)
-    except nilmdb.client.ClientError as e:
-        self.die("error creating stream: %s", str(e))

nilmdb/cmdline/destroy.py

@@ -1,25 +0,0 @@
-from nilmdb.utils.printf import *
-import nilmdb
-import nilmdb.client
-
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-def setup(self, sub):
-    cmd = sub.add_parser("destroy", help="Delete a stream and all data",
-                         formatter_class = def_form,
-                         description="""
-                         Destroy the stream at the specified path.  All
-                         data and metadata related to the stream is
-                         permanently deleted.
-                         """)
-    cmd.set_defaults(handler = cmd_destroy)
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("path",
-                       help="Path of the stream to delete, e.g. /foo/bar")
-
-def cmd_destroy(self):
-    """Destroy stream"""
-    try:
-        self.client.stream_destroy(self.args.path)
-    except nilmdb.client.ClientError as e:
-        self.die("error destroying stream: %s", str(e))

nilmdb/cmdline/extract.py

@@ -1,72 +0,0 @@
-from __future__ import print_function
-from nilmdb.utils.printf import *
-import nilmdb.client
-
-def setup(self, sub):
-    cmd = sub.add_parser("extract", help="Extract data",
-                         description="""
-                         Extract data from a stream.
-                         """)
-    cmd.set_defaults(verify = cmd_extract_verify,
-                     handler = cmd_extract)
-
-    group = cmd.add_argument_group("Data selection")
-    group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
-    group.add_argument("-s", "--start", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)")
-    group.add_argument("-e", "--end", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)")
-
-    group = cmd.add_argument_group("Output format")
-    group.add_argument("-b", "--bare", action="store_true",
-                       help="Exclude timestamps from output lines")
-    group.add_argument("-a", "--annotate", action="store_true",
-                       help="Include comments with some information "
-                       "about the stream")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps in annotated information")
-    group.add_argument("-c", "--count", action="store_true",
-                       help="Just output a count of matched data points")
-
-def cmd_extract_verify(self):
-    if self.args.start is not None and self.args.end is not None:
-        if self.args.start > self.args.end:
-            self.parser.error("start is after end")
-
-def cmd_extract(self):
-    streams = self.client.stream_list(self.args.path)
-    if len(streams) != 1:
-        self.die("error getting stream info for path %s", self.args.path)
-    layout = streams[0][1]
-
-    if self.args.timestamp_raw:
-        time_string = repr
-    else:
-        time_string = self.time_string
-
-    if self.args.annotate:
-        printf("# path: %s\n", self.args.path)
-        printf("# layout: %s\n", layout)
-        printf("# start: %s\n", time_string(self.args.start))
-        printf("# end: %s\n", time_string(self.args.end))
-
-    printed = False
-    for dataline in self.client.stream_extract(self.args.path,
-                                               self.args.start,
-                                               self.args.end,
-                                               self.args.count):
-        if self.args.bare and not self.args.count:
-            # Strip timestamp (first element).  Doesn't make sense
-            # if we are only returning a count.
-            dataline = ' '.join(dataline.split(' ')[1:])
-        print(dataline)
-        printed = True
-    if not printed:
-        if self.args.annotate:
-            printf("# no data\n")
-        return 2
-
-    return 0

nilmdb/cmdline/info.py

@@ -1,24 +0,0 @@
-import nilmdb
-from nilmdb.utils.printf import *
-from nilmdb.utils import human_size
-
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-def setup(self, sub):
-    cmd = sub.add_parser("info", help="Server information",
-                         formatter_class = def_form,
-                         description="""
-                         List information about the server, like
-                         version.
-                         """)
-    cmd.set_defaults(handler = cmd_info)
-
-def cmd_info(self):
-    """Print info about the server"""
-    printf("Client version: %s\n", nilmdb.__version__)
-    printf("Server version: %s\n", self.client.version())
-    printf("Server URL: %s\n", self.client.geturl())
-    dbinfo = self.client.dbinfo()
-    printf("Server database path: %s\n", dbinfo["path"])
-    printf("Server database size: %s\n", human_size(dbinfo["size"]))
-    printf("Server database free space: %s\n", human_size(dbinfo["free"]))

nilmdb/cmdline/insert.py

@@ -1,103 +0,0 @@
-from nilmdb.utils.printf import *
-import nilmdb
-import nilmdb.client
-import nilmdb.utils.timestamper as timestamper
-
-import sys
-
-def setup(self, sub):
-    cmd = sub.add_parser("insert", help="Insert data",
-                         description="""
-                         Insert data into a stream.
-                         """)
-    cmd.set_defaults(handler = cmd_insert)
-    cmd.add_argument("-q", "--quiet", action='store_true',
-                     help='suppress unnecessary messages')
-
-    group = cmd.add_argument_group("Timestamping",
-                                   description="""
-                                   If timestamps are already provided in the
-                                   input date, use --none.  Otherwise,
-                                   provide --start, or use --filename to
-                                   try to deduce timestamps from the file.
-
-                                   Set the TZ environment variable to change
-                                   the default timezone.
-                                   """)
-
-    group.add_argument("-r", "--rate", type=float,
-                       help="""
-                       If needed, rate in Hz (required when using --start)
-                       """)
-    exc = group.add_mutually_exclusive_group()
-    exc.add_argument("-s", "--start",
-                     metavar="TIME", type=self.arg_time,
-                     help="Starting timestamp (free-form)")
-    exc.add_argument("-f", "--filename", action="store_true",
-                     help="""
-                     Use filenames to determine start time
-                     (default, if filenames are provided)
-                     """)
-    exc.add_argument("-n", "--none", action="store_true",
-                     help="Timestamp is already present, don't add one")
-
-    group = cmd.add_argument_group("Required parameters")
-    group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
-    group.add_argument("file", nargs="*", default=['-'],
-                       help="File(s) to insert (default: - (stdin))")
-
-def cmd_insert(self):
-    # Find requested stream
-    streams = self.client.stream_list(self.args.path)
-    if len(streams) != 1:
-        self.die("error getting stream info for path %s", self.args.path)
-
-    if self.args.start and len(self.args.file) != 1:
-        self.die("error: --start can only be used with one input file")
-
-    for filename in self.args.file:
-        if filename == '-':
-            infile = sys.stdin
-        else:
-            try:
-                infile = open(filename, "r")
-            except IOError:
-                self.die("error opening input file %s", filename)
-
-        # Build a timestamper for this file
-        if self.args.none:
-            ts = timestamper.TimestamperNull(infile)
-        else:
-            if self.args.start:
-                start = self.args.start
-            else:
-                try:
-                    start = self.parse_time(filename)
-                except ValueError:
-                    self.die("error extracting time from filename '%s'",
-                             filename)
-
-            if not self.args.rate:
-                self.die("error: --rate is needed, but was not specified")
-            rate = self.args.rate
-
-            ts = timestamper.TimestamperRate(infile, start, rate)
-
-        # Print info
-        if not self.args.quiet:
-            printf("Input file: %s\n", filename)
-            printf("Timestamper: %s\n", str(ts))
-
-        # Insert the data
-        try:
-            self.client.stream_insert(self.args.path, ts)
-        except nilmdb.client.Error as e:
-            # TODO: It would be nice to be able to offer better errors
-            # here, particularly in the case of overlap, which just shows
-            # ugly bracketed ranges of 16-digit numbers and a mangled URL.
-            # Need to consider adding something like e.prettyprint()
-            # that is smarter about the contents of the error.
-            self.die("error inserting data: %s", str(e))
-
-    return

nilmdb/cmdline/list.py

@@ -1,77 +0,0 @@
-from nilmdb.utils.printf import *
-
-import fnmatch
-import argparse
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-def setup(self, sub):
-    cmd = sub.add_parser("list", help="List streams",
-                         formatter_class = def_form,
-                         description="""
-                         List streams available in the database,
-                         optionally filtering by layout or path.  Wildcards
-                         are accepted.
-                         """)
-    cmd.set_defaults(verify = cmd_list_verify,
-                     handler = cmd_list)
-
-    group = cmd.add_argument_group("Stream filtering")
-    group.add_argument("-p", "--path", metavar="PATH", default="*",
-                       help="Match only this path (-p can be omitted)")
-    group.add_argument("path_positional", default="*",
-                       nargs="?", help=argparse.SUPPRESS)
-    group.add_argument("-l", "--layout", default="*",
-                       help="Match only this stream layout")
-
-    group = cmd.add_argument_group("Interval details")
-    group.add_argument("-d", "--detail", action="store_true",
-                       help="Show available data time intervals")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps in time intervals")
-    group.add_argument("-s", "--start",
-                       metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)")
-    group.add_argument("-e", "--end",
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)")
-
-def cmd_list_verify(self):
-    # A hidden "path_positional" argument lets the user leave off the
-    # "-p" when specifying the path.  Handle it here.
-    got_opt = self.args.path != "*"
-    got_pos = self.args.path_positional != "*"
-    if got_pos:
-        if got_opt:
-            self.parser.error("too many paths specified")
-        else:
-            self.args.path = self.args.path_positional
-
-    if self.args.start is not None and self.args.end is not None:
-        if self.args.start >= self.args.end:
-            self.parser.error("start must precede end")
-
-def cmd_list(self):
-    """List available streams"""
-    streams = self.client.stream_list()
-
-    if self.args.timestamp_raw:
-        time_string = repr
-    else:
-        time_string = self.time_string
-
-    for (path, layout) in streams:
-        if not (fnmatch.fnmatch(path, self.args.path) and
-                fnmatch.fnmatch(layout, self.args.layout)):
-            continue
-
-        printf("%s %s\n", path, layout)
-        if not self.args.detail:
-            continue
-
-        printed = False
-        for (start, end) in self.client.stream_intervals(path, self.args.start,
-                                                         self.args.end):
-            printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
-            printed = True
-        if not printed:
-            printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -1,65 +0,0 @@
-from nilmdb.utils.printf import *
-import nilmdb
-import nilmdb.client
-
-def setup(self, sub):
-    cmd = sub.add_parser("metadata", help="Get or set stream metadata",
-                         description="""
-                         Get or set key=value metadata associated with
-                         a stream.
-                         """,
-                         usage="%(prog)s path [-g [key ...] | "
-                         "-s key=value [...] | -u key=value [...]]")
-    cmd.set_defaults(handler = cmd_metadata)
-
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
-
-    group = cmd.add_argument_group("Actions")
-    exc = group.add_mutually_exclusive_group()
-    exc.add_argument("-g", "--get", nargs="*", metavar="key",
-                     help="Get metadata for specified keys (default all)")
-    exc.add_argument("-s", "--set", nargs="+", metavar="key=value",
-                     help="Replace all metadata with provided "
-                     "key=value pairs")
-    exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
-                     help="Update metadata using provided "
-                     "key=value pairs")
-
-def cmd_metadata(self):
-    """Manipulate metadata"""
-    if self.args.set is not None or self.args.update is not None:
-        # Either set, or update
-        if self.args.set is not None:
-            keyvals = self.args.set
-            handler = self.client.stream_set_metadata
-        else:
-            keyvals = self.args.update
-            handler = self.client.stream_update_metadata
-
-        # Extract key=value pairs
-        data = {}
-        for keyval in keyvals:
-            kv = keyval.split('=')
-            if len(kv) != 2 or kv[0] == "":
-                self.die("error parsing key=value argument '%s'", keyval)
-            data[kv[0]] = kv[1]
-
-        # Make the call
-        try:
-            handler(self.args.path, data)
-        except nilmdb.client.ClientError as e:
-            self.die("error setting/updating metadata: %s", str(e))
-    else:
-        # Get (or unspecified)
-        keys = self.args.get or None
-        try:
-            data = self.client.stream_get_metadata(self.args.path, keys)
-        except nilmdb.client.ClientError as e:
-            self.die("error getting metadata: %s", str(e))
-        for key, value in sorted(data.items()):
-            # Omit nonexistant keys
-            if value is None:
-                value = ""
-            printf("%s=%s\n", key, value)

nilmdb/cmdline/remove.py

@@ -1,37 +0,0 @@
-from nilmdb.utils.printf import *
-import nilmdb
-import nilmdb.client
-
-def setup(self, sub):
-    cmd = sub.add_parser("remove", help="Remove data",
-                         description="""
-                         Remove all data from a specified time range within a
-                         stream.
-                         """)
-    cmd.set_defaults(handler = cmd_remove)
-
-    group = cmd.add_argument_group("Data selection")
-    group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar")
-    group.add_argument("-s", "--start", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)")
-    group.add_argument("-e", "--end", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)")
-
-    group = cmd.add_argument_group("Output format")
-    group.add_argument("-c", "--count", action="store_true",
-                       help="Output number of data points removed")
-
-def cmd_remove(self):
-    try:
-        count = self.client.stream_remove(self.args.path,
-                                          self.args.start, self.args.end)
-    except nilmdb.client.ClientError as e:
-        self.die("error removing data: %s", str(e))
-
-    if self.args.count:
-        printf("%d\n", count)
-
-    return 0

nilmdb/fileinterval.py

@@ -0,0 +1,37 @@
+"""FileInterval
+
+An Interval that is backed with file data storage"""
+
+from nilmdb.interval import Interval, IntervalSet, IntervalError
+from datetime import datetime
+import bisect
+
+class FileInterval(Interval):
+    """Represents an interval of time and its corresponding data"""
+
+    def __init__(self, start, end, 
+                 filename, 
+                 start_offset = None, end_offset = None):
+        self.start = start
+        self.end = end
+        self.filename = filename
+        if start_offset is None:
+            start_offset = 0
+        self.start_offset = start_offset
+        if end_offset is None:
+            f = open(filename, 'rb')
+            f.seek(0, os.SEEK_END)
+            end_offset = f.tell()
+        self.end_offset = end_offset
+
+    def __setattr__(self, name, value):
+        pass
+
+    def subset(self, start, end):
+        """Return a new Interval that is a subset of this one"""
+        # TODO: Any magic regarding file/offset/length mapping for subsets
+        if (start < self.start or end > self.end):
+            raise IntervalError("not a subset")
+        return FileInterval(start, end)            
+
+    

nilmdb/interval.py

@@ -0,0 +1,205 @@
+"""Interval and IntervalSet
+
+Represents an interval of time, and a sorted set of such intervals"""
+
+from datetime import datetime
+import bisect
+
+class IntervalError(Exception):
+    """Error due to interval overlap, etc"""
+    pass
+
+class Interval(object):
+    """Represents an interval of time"""
+
+    start = None
+    end = None
+
+    def __init__(self, start, end):
+        self.start = start
+        self.end = end
+
+    def __repr__(self):
+        return "Interval(" + repr(self.start) + ", " + repr(self.end) + ")"
+
+    def __str__(self):
+        return "[" + str(self.start) + " -> " + str(self.end) + "]"
+
+    def __setattr__(self, name, value):
+        """Set attribute"""
+        # TODO: If we need to manipulate file names, offsets, lengths, etc,
+        # based on start and end time changing, maybe this is the right spot?
+        # Or we could just disallow changing it here.
+        if not isinstance(value, datetime):
+            raise IntervalError("Must set datetime values")
+        self.__dict__[name] = value
+        if (type(self.start) is type(self.end)):
+            if (self.start > self.end):
+                raise IntervalError("Interval start must precede interval end")
+
+    def __cmp__(self, other):
+        """Compare two intervals.  If non-equal, order by start then end"""
+        if not isinstance(other, Interval):
+            raise TypeError("Can't compare to non-interval")
+        if (self.start == other.start):
+            if (self.end < other.end):
+                return -1
+            if (self.end > other.end):
+                return 1
+            return 0
+        if (self.start < other.start):
+            return -1
+        return 1
+                
+    def intersects(self, other):
+        """Return True if two Interval objects intersect"""
+        if (not isinstance(other, Interval)):
+            raise TypeError("need Interval for intersection test")
+        if (self.end <= other.start or
+            self.start >= other.end):
+            return False
+        else:
+            return True
+
+    def is_adjacent(self, other):
+        """Return True if two Intervals are adjacent (same end or start)"""
+        if (not isinstance(other, Interval)):
+            raise TypeError("need Interval for adjacency test")
+        if (self.end == other.start or
+            self.start == other.end):
+            return True
+        else:
+            return False
+
+    def subset(self, start, end):
+        """Return a new Interval that is a subset of this one"""
+        # TODO: Any magic regarding file/offset/length mapping for subsets
+        if (start < self.start or end > self.end):
+            raise IntervalError("not a subset")
+        return Interval(start, end)            
+
+class IntervalSet(object):
+    """A non-intersecting set of intervals
+
+    Kept sorted internally"""
+
+    def __init__(self, iterable=None):
+        self.data = []
+        if iterable is not None:
+            if isinstance(iterable, Interval):
+                iterable = [iterable]
+            self._add_intervals(iterable)
+
+    def __iter__(self):
+        return self.data.__iter__()
+
+    def __repr__(self):
+        return "IntervalSet(" + repr(list(self.data)) + ")"
+
+    def __cmp__(self, other):
+        # compare isn't supported, they don't really have an ordering
+        raise TypeError("can't compare IntervalSets with cmp()")
+
+    def __eq__(self, other):
+        """Test equality of two IntervalSets.
+
+        Treats adjacent Intervals as equivalent to one long interval,
+        so this function really tests whether the IntervalSets cover
+        the same spans of time."""
+        if not isinstance(other, IntervalSet):
+            return False
+        i = 0
+        j = 0
+        outside = True
+        try:
+            while True:
+                if (outside):
+                    # To match, we need to be finished this set
+                    if (i >= len(self) and j >= len(other)):
+                        return True
+                    # Or the starts need to match
+                    if (self[i].start != other[j].start):
+                        return False
+                    outside = False
+                else:
+                    # We can move on if the two interval ends match
+                    if (self[i].end == other[j].end):
+                        i += 1
+                        j += 1
+                        outside = True
+                    else:
+                        # Whichever ends first needs to be adjacent to the next
+                        if (self[i].end < other[j].end):
+                            if (not self[i].is_adjacent(self[i+1])):
+                                return False
+                            i += 1
+                        else:
+                            if (not other[j].is_adjacent(other[j+1])):
+                                return False
+                            j += 1
+        except IndexError:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, key):
+        return self.data.__getitem__(key)
+
+    def __iadd__(self, other):
+        """Inplace add -- modifies self
+
+        This throws an exception if the regions being added intersect."""
+        if isinstance(other, Interval):
+            other = [other]
+        self._add_intervals(other)
+        return self
+        
+    def __add__(self, other):
+        """Add -- returns a new object
+
+        This throws an exception if the regions being added intersect."""
+        new = IntervalSet(self)
+        new += IntervalSet(other)
+        return new
+
+    def __and__(self, other):
+        """Compute a new IntervalSet from the intersection of two others
+
+        Output intervals are built as subsets of the intervals in the
+        first argument (self)."""
+        # If we were given a set, intersect with each interval in that set
+        if isinstance(other, IntervalSet):
+            out = IntervalSet()
+            for interval in other.data:
+                out += self & interval
+            return out
+
+        if not isinstance(other, Interval):
+            raise TypeError("can't intersect with that type")
+
+        out = IntervalSet()
+        for this in self.data:
+            # If there's any overlap, add the overlapping region
+            if (this.end > other.start and this.start < other.end):
+                out += this.subset(max(this.start, other.start),
+                                   min(this.end, other.end))
+        return out
+    
+    def _add_intervals(self, iterable):
+        """Add each Interval from an interable to this set"""
+        for element in iter(iterable):
+            self._add_single_interval(element)
+
+    def _add_single_interval(self, interval):
+        """Add one Interval to this set"""
+        if (not isinstance(interval, Interval)):
+            raise TypeError("can only add Intervals")
+        for existing in self.data:
+            if existing.intersects(interval):
+                raise IntervalError("Tried to add overlapping interval "
+                                    "to this set")
+        bisect.insort(self.data, interval)

nilmdb/scripts/__init__.py

@@ -1 +0,0 @@
-# Command line scripts

nilmdb/scripts/nilmdb_server.py

@@ -1,81 +0,0 @@
-#!/usr/bin/python
-
-import nilmdb.server
-import argparse
-import os
-import socket
-
-def main():
-    """Main entry point for the 'nilmdb-server' command line script"""
-
-    parser = argparse.ArgumentParser(
-        description = 'Run the NilmDB server',
-        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
-
-    parser.add_argument("-V", "--version", action="version",
-                        version = nilmdb.__version__)
-
-    group = parser.add_argument_group("Standard options")
-    group.add_argument('-a', '--address',
-                       help = 'Only listen on the given address',
-                       default = '0.0.0.0')
-    group.add_argument('-p', '--port', help = 'Listen on the given port',
-                       type = int, default = 12380)
-    group.add_argument('-d', '--database', help = 'Database directory',
-                       default = os.path.join(os.getcwd(), "db"))
-    group.add_argument('-q', '--quiet', help = 'Silence output',
-                       action = 'store_true')
-
-    group = parser.add_argument_group("Debug options")
-    group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and '
-                       'invoke interactive shell afterwards',
-                       action = 'store_true')
-
-    args = parser.parse_args()
-
-    # Create database object
-    db = nilmdb.server.NilmDB(args.database)
-
-    # Configure the server
-    if args.quiet:
-        embedded = True
-    else:
-        embedded = False
-    server = nilmdb.server.Server(db,
-                                  host = args.address,
-                                  port = args.port,
-                                  embedded = embedded)
-
-    # Print info
-    if not args.quiet:
-        print "Database: %s" % (os.path.realpath(args.database))
-        if args.address == '0.0.0.0' or args.address == '::':
-            host = socket.getfqdn()
-        else:
-            host = args.address
-        print "Server URL: http://%s:%d/" % ( host, args.port)
-        print "----"
-
-    # Run it
-    if args.yappi:
-        print "Running in yappi"
-        try:
-            import yappi
-            yappi.start()
-            server.start(blocking = True)
-        finally:
-            yappi.stop()
-            yappi.print_stats(sort_type = yappi.SORTTYPE_TTOT, limit = 50)
-            from IPython import embed
-            embed(header = "Use the yappi object to explore further, "
-                  "quit to exit")
-    else:
-        server.start(blocking = True)
-
-    # Clean up
-    if not args.quiet:
-        print "Closing database"
-        db.close()
-
-if __name__ == "__main__":
-    main()

nilmdb/scripts/nilmtool.py

@@ -1,10 +0,0 @@
-#!/usr/bin/python
-
-import nilmdb.cmdline
-
-def main():
-    """Main entry point for the 'nilmtool' command line script"""
-    nilmdb.cmdline.Cmdline().run()
-
-if __name__ == "__main__":
-    main()

nilmdb/server/__init__.py

@@ -1,22 +0,0 @@
-"""nilmdb.server"""
-
-from __future__ import absolute_import
-
-# Try to set up pyximport to automatically rebuild Cython modules.  If
-# this doesn't work, it's OK, as long as the modules were built externally.
-# (e.g. python setup.py build_ext --inplace)
-try:
-    import Cython
-    import distutils.version
-    if (distutils.version.LooseVersion(Cython.__version__) <
-        distutils.version.LooseVersion("0.16")): # pragma: no cover
-        raise ImportError("Cython version too old")
-    import pyximport
-    pyximport.install(inplace = True, build_in_temp = False)
-except ImportError: # pragma: no cover
-    pass
-
-import nilmdb.server.layout
-from nilmdb.server.nilmdb import NilmDB
-from nilmdb.server.server import Server
-from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

nilmdb/server/bulkdata.py

@@ -1,488 +0,0 @@
-# Fixed record size bulk data storage
-
-# Need absolute_import so that "import nilmdb" won't pull in
-# nilmdb.py, but will pull the parent nilmdb module instead.
-from __future__ import absolute_import
-from __future__ import division
-import nilmdb
-from nilmdb.utils.printf import *
-
-import os
-import cPickle as pickle
-import struct
-import mmap
-import re
-
-# If we have the faulthandler module, use it.  All of the mmap stuff
-# might trigger a SIGSEGV or SIGBUS if we're not careful, and
-# faulthandler will give a traceback in that case.  (the Python
-# interpreter will still die either way).
-try: # pragma: no cover
-    import faulthandler
-    faulthandler.enable()
-except: # pragma: no cover
-    pass
-
-# Up to 256 open file descriptors at any given time.
-# These variables are global so they can be used in the decorator arguments.
-table_cache_size = 16
-fd_cache_size = 16
-
-@nilmdb.utils.must_close(wrap_verify = True)
-class BulkData(object):
-    def __init__(self, basepath, **kwargs):
-        self.basepath = basepath
-        self.root = os.path.join(self.basepath, "data")
-
-        # Tuneables
-        if "file_size" in kwargs:
-            self.file_size = kwargs["file_size"]
-        else:
-            # Default to approximately 128 MiB per file
-            self.file_size = 128 * 1024 * 1024
-
-        if "files_per_dir" in kwargs:
-            self.files_per_dir = kwargs["files_per_dir"]
-        else:
-            # 32768 files per dir should work even on FAT32
-            self.files_per_dir = 32768
-
-        # Make root path
-        if not os.path.isdir(self.root):
-            os.mkdir(self.root)
-
-    def close(self):
-        self.getnode.cache_remove_all()
-
-    def _encode_filename(self, path):
-        # Encode all paths to UTF-8, regardless of sys.getfilesystemencoding(),
-        # because we want to be able to represent all code points and the user
-        # will never be directly exposed to filenames.  We can then do path
-        # manipulations on the UTF-8 directly.
-        if isinstance(path, unicode):
-            return path.encode('utf-8')
-        return path
-
-    def create(self, unicodepath, layout_name):
-        """
-        unicodepath: path to the data (e.g. u'/newton/prep').
-        Paths must contain at least two elements, e.g.:
-           /newton/prep
-           /newton/raw
-           /newton/upstairs/prep
-           /newton/upstairs/raw
-
-        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
-        """
-        path = self._encode_filename(unicodepath)
-
-        if path[0] != '/':
-            raise ValueError("paths must start with /")
-        [ group, node ] = path.rsplit("/", 1)
-        if group == '':
-            raise ValueError("invalid path; path must contain at least one "
-                             "folder")
-
-        # Get layout, and build format string for struct module
-        try:
-            layout = nilmdb.server.layout.get_named(layout_name)
-            struct_fmt = '<d'  # Little endian, double timestamp
-            struct_mapping = {
-                "int8": 'b',
-                "uint8": 'B',
-                "int16": 'h',
-                "uint16": 'H',
-                "int32": 'i',
-                "uint32": 'I',
-                "int64": 'q',
-                "uint64": 'Q',
-                "float32": 'f',
-                "float64": 'd',
-                }
-            struct_fmt += struct_mapping[layout.datatype] * layout.count
-        except KeyError:
-            raise ValueError("no such layout, or bad data types")
-
-        # Create the table.  Note that we make a distinction here
-        # between NilmDB paths (always Unix style, split apart
-        # manually) and OS paths (built up with os.path.join)
-
-        # Make directories leading up to this one
-        elements = path.lstrip('/').split('/')
-        for i in range(len(elements)):
-            ospath = os.path.join(self.root, *elements[0:i])
-            if Table.exists(ospath):
-                raise ValueError("path is subdir of existing node")
-            if not os.path.isdir(ospath):
-                os.mkdir(ospath)
-
-        # Make the final dir
-        ospath = os.path.join(self.root, *elements)
-        if os.path.isdir(ospath):
-            raise ValueError("subdirs of this path already exist")
-        os.mkdir(ospath)
-
-        # Write format string to file
-        Table.create(ospath, struct_fmt, self.file_size, self.files_per_dir)
-
-        # Open and cache it
-        self.getnode(unicodepath)
-
-        # Success
-        return
-
-    def destroy(self, unicodepath):
-        """Fully remove all data at a particular path.  No way to undo
-        it!  The group/path structure is removed, too."""
-        path = self._encode_filename(unicodepath)
-
-        # Get OS path
-        elements = path.lstrip('/').split('/')
-        ospath = os.path.join(self.root, *elements)
-
-        # Remove Table object from cache
-        self.getnode.cache_remove(self, unicodepath)
-
-        # Remove the contents of the target directory
-        if not Table.exists(ospath):
-            raise ValueError("nothing at that path")
-        for (root, dirs, files) in os.walk(ospath, topdown = False):
-            for name in files:
-                os.remove(os.path.join(root, name))
-            for name in dirs:
-                os.rmdir(os.path.join(root, name))
-
-        # Remove empty parent directories
-        for i in reversed(range(len(elements))):
-            ospath = os.path.join(self.root, *elements[0:i+1])
-            try:
-                os.rmdir(ospath)
-            except OSError:
-                break
-
-    # Cache open tables
-    @nilmdb.utils.lru_cache(size = table_cache_size,
-                            onremove = lambda x: x.close())
-    def getnode(self, unicodepath):
-        """Return a Table object corresponding to the given database
-        path, which must exist."""
-        path = self._encode_filename(unicodepath)
-        elements = path.lstrip('/').split('/')
-        ospath = os.path.join(self.root, *elements)
-        return Table(ospath)
-
-@nilmdb.utils.must_close(wrap_verify = True)
-class File(object):
-    """Object representing a single file on disk.  Data can be appended,
-    or the self.mmap handle can be used for random reads."""
-
-    def __init__(self, root, subdir, filename):
-        # Create path if it doesn't exist
-        try:
-            os.mkdir(os.path.join(root, subdir))
-        except OSError:
-            pass
-
-        # Open/create file
-        self._f = open(os.path.join(root, subdir, filename), "a+b", 0)
-
-        # Seek to end, and get size
-        self._f.seek(0, 2)
-        self.size = self._f.tell()
-
-        # Open mmap object
-        self.mmap = None
-        self._mmap_reopen()
-
-    def _mmap_reopen(self):
-        if self.size == 0:
-            # Don't mmap if the file is empty; it would fail
-            pass
-        elif self.mmap is None:
-            # Not opened yet, so open it
-            self.mmap = mmap.mmap(self._f.fileno(), 0)
-        else:
-            # Already opened, so just resize it
-            self.mmap.resize(self.size)
-
-    def close(self):
-        if self.mmap is not None:
-            self.mmap.close()
-        self._f.close()
-
-    def append(self, data):
-        # Write data, flush it, and resize our mmap accordingly
-        self._f.write(data)
-        self._f.flush()
-        self.size += len(data)
-        self._mmap_reopen()
-
-@nilmdb.utils.must_close(wrap_verify = True)
-class Table(object):
-    """Tools to help access a single table (data at a specific OS path)."""
-    # See design.md for design details
-
-    # Class methods, to help keep format details in this class.
-    @classmethod
-    def exists(cls, root):
-        """Return True if a table appears to exist at this OS path"""
-        return os.path.isfile(os.path.join(root, "_format"))
-
-    @classmethod
-    def create(cls, root, struct_fmt, file_size, files_per_dir):
-        """Initialize a table at the given OS path.
-        'struct_fmt' is a Struct module format description"""
-
-        # Calculate rows per file so that each file is approximately
-        # file_size bytes.
-        packer = struct.Struct(struct_fmt)
-        rows_per_file = max(file_size // packer.size, 1)
-
-        fmt = { "rows_per_file": rows_per_file,
-                "files_per_dir": files_per_dir,
-                "struct_fmt": struct_fmt,
-                "version": 1 }
-        with open(os.path.join(root, "_format"), "wb") as f:
-            pickle.dump(fmt, f, 2)
-
-    # Normal methods
-    def __init__(self, root):
-        """'root' is the full OS path to the directory of this table"""
-        self.root = root
-
-        # Load the format and build packer
-        with open(os.path.join(self.root, "_format"), "rb") as f:
-            fmt = pickle.load(f)
-
-        if fmt["version"] != 1: # pragma: no cover (just future proofing)
-            raise NotImplementedError("version " + fmt["version"] +
-                                      " bulk data store not supported")
-
-        self.rows_per_file = fmt["rows_per_file"]
-        self.files_per_dir = fmt["files_per_dir"]
-        self.packer = struct.Struct(fmt["struct_fmt"])
-        self.file_size = self.packer.size * self.rows_per_file
-
-        # Find nrows
-        self.nrows = self._get_nrows()
-
-    def close(self):
-        self.file_open.cache_remove_all()
-
-    # Internal helpers
-    def _get_nrows(self):
-        """Find nrows by locating the lexicographically last filename
-        and using its size"""
-        # Note that this just finds a 'nrows' that is guaranteed to be
-        # greater than the row number of any piece of data that
-        # currently exists, not necessarily all data that _ever_
-        # existed.
-        regex = re.compile("^[0-9a-f]{4,}$")
-
-        # Find the last directory.  We sort and loop through all of them,
-        # starting with the numerically greatest, because the dirs could be
-        # empty if something was deleted.
-        subdirs = sorted(filter(regex.search, os.listdir(self.root)),
-                         key = lambda x: int(x, 16), reverse = True)
-
-        for subdir in subdirs:
-            # Now find the last file in that dir
-            path = os.path.join(self.root, subdir)
-            files = filter(regex.search, os.listdir(path))
-            if not files: # pragma: no cover (shouldn't occur)
-                # Empty dir: try the next one
-                continue
-
-            # Find the numerical max
-            filename = max(files, key = lambda x: int(x, 16))
-            offset = os.path.getsize(os.path.join(self.root, subdir, filename))
-
-            # Convert to row number
-            return self._row_from_offset(subdir, filename, offset)
-
-        # No files, so no data
-        return 0
-
-    def _offset_from_row(self, row):
-        """Return a (subdir, filename, offset, count) tuple:
-
-          subdir: subdirectory for the file
-        filename: the filename that contains the specified row
-          offset: byte offset of the specified row within the file
-           count: number of rows (starting at offset) that fit in the file
-        """
-        filenum = row // self.rows_per_file
-        # It's OK if these format specifiers are too short; the filenames
-        # will just get longer but will still sort correctly.
-        dirname = sprintf("%04x", filenum // self.files_per_dir)
-        filename = sprintf("%04x", filenum % self.files_per_dir)
-        offset = (row % self.rows_per_file) * self.packer.size
-        count = self.rows_per_file - (row % self.rows_per_file)
-        return (dirname, filename, offset, count)
-
-    def _row_from_offset(self, subdir, filename, offset):
-        """Return the row number that corresponds to the given
-        'subdir/filename' and byte-offset within that file."""
-        if (offset % self.packer.size) != 0: # pragma: no cover; shouldn't occur
-            raise ValueError("file offset is not a multiple of data size")
-        filenum = int(subdir, 16) * self.files_per_dir + int(filename, 16)
-        row = (filenum * self.rows_per_file) + (offset // self.packer.size)
-        return row
-
-    # Cache open files
-    @nilmdb.utils.lru_cache(size = fd_cache_size,
-                            onremove = lambda f: f.close())
-    def file_open(self, subdir, filename):
-        """Open and map a given 'subdir/filename' (relative to self.root).
-        Will be automatically closed when evicted from the cache."""
-        return File(self.root, subdir, filename)
-
-    def append(self, data):
-        """Append the data and flush it to disk.
-        data is a nested Python list [[row],[row],[...]]"""
-        remaining = len(data)
-        dataiter = iter(data)
-        while remaining:
-            # See how many rows we can fit into the current file, and open it
-            (subdir, fname, offset, count) = self._offset_from_row(self.nrows)
-            if count > remaining:
-                count = remaining
-
-            f = self.file_open(subdir, fname)
-
-            # Write the data
-            for i in xrange(count):
-                row = dataiter.next()
-                f.append(self.packer.pack(*row))
-            remaining -= count
-            self.nrows += count
-
-    def __getitem__(self, key):
-        """Extract data and return it.  Supports simple indexing
-        (table[n]) and range slices (table[n:m]).  Returns a nested
-        Python list [[row],[row],[...]]"""
-
-        # Handle simple slices
-        if isinstance(key, slice):
-            # Fall back to brute force if the slice isn't simple
-            if ((key.step is not None and key.step != 1) or
-                key.start is None or
-                key.stop is None or
-                key.start >= key.stop or
-                key.start < 0 or
-                key.stop > self.nrows):
-                return [ self[x] for x in xrange(*key.indices(self.nrows)) ]
-
-            ret = []
-            row = key.start
-            remaining = key.stop - key.start
-            while remaining:
-                (subdir, filename, offset, count) = self._offset_from_row(row)
-                if count > remaining:
-                    count = remaining
-                mm = self.file_open(subdir, filename).mmap
-                for i in xrange(count):
-                    ret.append(list(self.packer.unpack_from(mm, offset)))
-                    offset += self.packer.size
-                remaining -= count
-                row += count
-            return ret
-
-        # Handle single points
-        if key < 0 or key >= self.nrows:
-            raise IndexError("Index out of range")
-        (subdir, filename, offset, count) = self._offset_from_row(key)
-        mm = self.file_open(subdir, filename).mmap
-        # unpack_from ignores the mmap object's current seek position
-        return list(self.packer.unpack_from(mm, offset))
-
-    def _remove_rows(self, subdir, filename, start, stop):
-        """Helper to mark specific rows as being removed from a
-        file, and potentially removing or truncating the file itself."""
-        # Import an existing list of deleted rows for this file
-        datafile = os.path.join(self.root, subdir, filename)
-        cachefile = datafile + ".removed"
-        try:
-            with open(cachefile, "rb") as f:
-                ranges = pickle.load(f)
-            cachefile_present = True
-        except:
-            ranges = []
-            cachefile_present = False
-
-        # Append our new range and sort
-        ranges.append((start, stop))
-        ranges.sort()
-
-        # Merge adjacent ranges into "out"
-        merged = []
-        prev = None
-        for new in ranges:
-            if prev is None:
-                # No previous range, so remember this one
-                prev = new
-            elif prev[1] == new[0]:
-                # Previous range connected to this new one; extend prev
-                prev = (prev[0], new[1])
-            else:
-                # Not connected; append previous and start again
-                merged.append(prev)
-                prev = new
-        if prev is not None:
-            merged.append(prev)
-
-        # If the range covered the whole file, we can delete it now.
-        # Note that the last file in a table may be only partially
-        # full (smaller than self.rows_per_file).  We purposely leave
-        # those files around rather than deleting them, because the
-        # remainder will be filled on a subsequent append(), and things
-        # are generally easier if we don't have to special-case that.
-        if (len(merged) == 1 and
-            merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
-            # Close potentially open file in file_open LRU cache
-            self.file_open.cache_remove(self, subdir, filename)
-
-            # Delete files
-            os.remove(datafile)
-            if cachefile_present:
-                os.remove(cachefile)
-
-            # Try deleting subdir, too
-            try:
-                os.rmdir(os.path.join(self.root, subdir))
-            except:
-                pass
-        else:
-            # Update cache.  Try to do it atomically.
-            nilmdb.utils.atomic.replace_file(cachefile,
-                                             pickle.dumps(merged, 2))
-
-    def remove(self, start, stop):
-        """Remove specified rows [start, stop) from this table.
-
-        If a file is left empty, it is fully removed.  Otherwise, a
-        parallel data file is used to remember which rows have been
-        removed, and the file is otherwise untouched."""
-        if start < 0 or start > stop or stop > self.nrows:
-            raise IndexError("Index out of range")
-
-        row = start
-        remaining = stop - start
-        while remaining:
-            # Loop through each file that we need to touch
-            (subdir, filename, offset, count) = self._offset_from_row(row)
-            if count > remaining:
-                count = remaining
-            row_offset = offset // self.packer.size
-            # Mark the rows as being removed
-            self._remove_rows(subdir, filename, row_offset, row_offset + count)
-            remaining -= count
-            row += count
-
-class TimestampOnlyTable(object):
-    """Helper that lets us pass a Tables object into bisect, by
-    returning only the timestamp when a particular row is requested."""
-    def __init__(self, table):
-        self.table = table
-    def __getitem__(self, index):
-        return self.table[index][0]

nilmdb/server/errors.py

@@ -1,12 +0,0 @@
-"""Exceptions"""
-
-class NilmDBError(Exception):
-    """Base exception for NilmDB errors"""
-    def __init__(self, message = "Unspecified error"):
-        Exception.__init__(self, message)
-
-class StreamError(NilmDBError):
-    pass
-
-class OverlapError(NilmDBError):
-    pass

nilmdb/server/interval.pyx

@@ -1,329 +0,0 @@
-"""Interval, IntervalSet
-
-Represents an interval of time, and a set of such intervals.
-
-Intervals are half-open, ie. they include data points with timestamps
-[start, end)
-"""
-
-# First implementation kept a sorted list of intervals and used
-# biesct() to optimize some operations, but this was too slow.
-
-# Second version was based on the quicksect implementation from
-# python-bx, modified slightly to handle floating point intervals.
-# This didn't support deletion.
-
-# Third version is more similar to the first version, using a rb-tree
-# instead of a simple sorted list to maintain O(log n) operations.
-
-# Fourth version is an optimized rb-tree that stores interval starts
-# and ends directly in the tree, like bxinterval did.
-
-cimport rbtree
-cdef extern from "stdint.h":
-    ctypedef unsigned long long uint64_t
-
-class IntervalError(Exception):
-    """Error due to interval overlap, etc"""
-    pass
-
-cdef class Interval:
-    """Represents an interval of time."""
-
-    cdef public double start, end
-
-    def __init__(self, double start, double end):
-        """
-        'start' and 'end' are arbitrary floats that represent time
-        """
-        if start >= end:
-            # Explicitly disallow zero-width intervals (since they're half-open)
-            raise IntervalError("start %s must precede end %s" % (start, end))
-        self.start = float(start)
-        self.end = float(end)
-
-    def __repr__(self):
-        s = repr(self.start) + ", " + repr(self.end)
-        return self.__class__.__name__ + "(" + s + ")"
-
-    def __str__(self):
-        return "[" + repr(self.start) + " -> " + repr(self.end) + ")"
-
-    def __cmp__(self, Interval other):
-        """Compare two intervals.  If non-equal, order by start then end"""
-        if not isinstance(other, Interval):
-            raise TypeError("bad type")
-        if self.start == other.start:
-            if self.end < other.end:
-                return -1
-            if self.end > other.end:
-                return 1
-            return 0
-        if self.start < other.start:
-            return -1
-        return 1
-
-    cpdef intersects(self, Interval other):
-        """Return True if two Interval objects intersect"""
-        if (self.end <= other.start or self.start >= other.end):
-            return False
-        return True
-
-    cpdef subset(self, double start, double end):
-        """Return a new Interval that is a subset of this one"""
-        # A subclass that tracks additional data might override this.
-        if start < self.start or end > self.end:
-            raise IntervalError("not a subset")
-        return Interval(start, end)
-
-cdef class DBInterval(Interval):
-    """
-    Like Interval, but also tracks corresponding start/end times and
-    positions within the database.  These are not currently modified
-    when subsets are taken, but can be used later to help zero in on
-    database positions.
-
-    The actual 'start' and 'end' will always fall within the database
-    start and end, e.g.:
-        db_start = 100, db_startpos = 10000
-        start = 123
-        end = 150
-        db_end = 200, db_endpos = 20000
-    """
-
-    cpdef public double db_start, db_end
-    cpdef public uint64_t db_startpos, db_endpos
-
-    def __init__(self, start, end,
-                 db_start, db_end,
-                 db_startpos, db_endpos):
-        """
-        'db_start' and 'db_end' are arbitrary floats that represent
-        time.  They must be a strict superset of the time interval
-        covered by 'start' and 'end'.  The 'db_startpos' and
-        'db_endpos' are arbitrary database position indicators that
-        correspond to those points.
-        """
-        Interval.__init__(self, start, end)
-        self.db_start = db_start
-        self.db_end = db_end
-        self.db_startpos = db_startpos
-        self.db_endpos = db_endpos
-        if db_start > start or db_end < end:
-            raise IntervalError("database times must span the interval times")
-
-    def __repr__(self):
-        s = repr(self.start) + ", " + repr(self.end)
-        s += ", " + repr(self.db_start) + ", " + repr(self.db_end)
-        s += ", " + repr(self.db_startpos) + ", " + repr(self.db_endpos)
-        return self.__class__.__name__ + "(" + s + ")"
-
-    cpdef subset(self, double start, double end):
-        """
-        Return a new DBInterval that is a subset of this one
-        """
-        if start < self.start or end > self.end:
-            raise IntervalError("not a subset")
-        return DBInterval(start, end,
-                          self.db_start, self.db_end,
-                          self.db_startpos, self.db_endpos)
-
-cdef class IntervalSet:
-    """
-    A non-intersecting set of intervals.
-    """
-
-    cdef public rbtree.RBTree tree
-
-    def __init__(self, source=None):
-        """
-        'source' is an Interval or IntervalSet to add.
-        """
-        self.tree = rbtree.RBTree()
-        if source is not None:
-            self += source
-
-    def __iter__(self):
-        for node in self.tree:
-            if node.obj:
-                yield node.obj
-
-    def __len__(self):
-        return sum(1 for x in self)
-
-    def __repr__(self):
-        descs = [ repr(x) for x in self ]
-        return self.__class__.__name__ + "([" + ", ".join(descs) + "])"
-
-    def __str__(self):
-        descs = [ str(x) for x in self ]
-        return  "[" + ", ".join(descs) + "]"
-
-    def __match__(self, other):
-        # This isn't particularly efficient, but it shouldn't get used in the
-        # general case.
-        """Test equality of two IntervalSets.
-
-        Treats adjacent Intervals as equivalent to one long interval,
-        so this function really tests whether the IntervalSets cover
-        the same spans of time."""
-        i = 0
-        j = 0
-        outside = True
-
-        def is_adjacent(a, b):
-            """Return True if two Intervals are adjacent (same end or start)"""
-            if a.end == b.start or b.end == a.start:
-                return True
-            else:
-                return False
-
-        this = list(self)
-        that = list(other)
-
-        try:
-            while True:
-                if (outside):
-                    # To match, we need to be finished both sets
-                    if (i >= len(this) and j >= len(that)):
-                        return True
-                    # Or the starts need to match
-                    if (this[i].start != that[j].start):
-                        return False
-                    outside = False
-                else:
-                    # We can move on if the two interval ends match
-                    if (this[i].end == that[j].end):
-                        i += 1
-                        j += 1
-                        outside = True
-                    else:
-                        # Whichever ends first needs to be adjacent to the next
-                        if (this[i].end < that[j].end):
-                            if (not is_adjacent(this[i],this[i+1])):
-                                return False
-                            i += 1
-                        else:
-                            if (not is_adjacent(that[j],that[j+1])):
-                                return False
-                            j += 1
-        except IndexError:
-            return False
-
-    # Use __richcmp__ instead of __eq__, __ne__ for Cython.
-    def __richcmp__(self, other, int op):
-        if op == 2: # ==
-            return self.__match__(other)
-        elif op == 3: # !=
-            return not self.__match__(other)
-        return False
-    #def __eq__(self, other):
-    #    return self.__match__(other)
-    #
-    #def __ne__(self, other):
-    #    return not self.__match__(other)
-
-    def __iadd__(self, object other not None):
-        """Inplace add -- modifies self
-
-        This throws an exception if the regions being added intersect."""
-        if isinstance(other, Interval):
-            if self.intersects(other):
-                raise IntervalError("Tried to add overlapping interval "
-                                    "to this set")
-            self.tree.insert(rbtree.RBNode(other.start, other.end, other))
-        else:
-            for x in other:
-                self.__iadd__(x)
-        return self
-
-    def iadd_nocheck(self, Interval other not None):
-        """Inplace add -- modifies self.
-        'Optimized' version that doesn't check for intersection and
-        only inserts the new interval into the tree."""
-        self.tree.insert(rbtree.RBNode(other.start, other.end, other))
-
-    def __isub__(self, Interval other not None):
-        """Inplace subtract -- modifies self
-
-        Removes an interval from the set.  Must exist exactly
-        as provided -- cannot remove a subset of an existing interval."""
-        i = self.tree.find(other.start, other.end)
-        if i is None:
-            raise IntervalError("interval " + str(other) + " not in tree")
-        self.tree.delete(i)
-        return self
-
-    def __add__(self, other not None):
-        """Add -- returns a new object"""
-        new = IntervalSet(self)
-        new += IntervalSet(other)
-        return new
-
-    def __and__(self, other not None):
-        """
-        Compute a new IntervalSet from the intersection of two others
-
-        Output intervals are built as subsets of the intervals in the
-        first argument (self).
-        """
-        out = IntervalSet()
-
-        if not isinstance(other, IntervalSet):
-            for i in self.intersection(other):
-                out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-        else:
-            for x in other:
-                for i in self.intersection(x):
-                    out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-
-        return out
-
-    def intersection(self, Interval interval not None, orig = False):
-        """
-        Compute a sequence of intervals that correspond to the
-        intersection between `self` and the provided interval.
-        Returns a generator that yields each of these intervals
-        in turn.
-
-        Output intervals are built as subsets of the intervals in the
-        first argument (self).
-
-        If orig = True, also return the original interval that was
-        (potentially) subsetted to make the one that is being
-        returned.
-        """
-        if not isinstance(interval, Interval):
-            raise TypeError("bad type")
-        for n in self.tree.intersect(interval.start, interval.end):
-            i = n.obj
-            if i:
-                if i.start >= interval.start and i.end <= interval.end:
-                    if orig:
-                        yield (i, i)
-                    else:
-                        yield i
-                else:
-                    subset = i.subset(max(i.start, interval.start),
-                                      min(i.end, interval.end))
-                    if orig:
-                        yield (subset, i)
-                    else:
-                        yield subset
-
-    cpdef intersects(self, Interval other):
-        """Return True if this IntervalSet intersects another interval"""
-        for n in self.tree.intersect(other.start, other.end):
-            if n.obj.intersects(other):
-                return True
-        return False
-
-    def find_end(self, double t):
-        """
-        Return an Interval from this tree that ends at time t, or
-        None if it doesn't exist.
-        """
-        n = self.tree.find_left_end(t)
-        if n and n.obj.end == t:
-            return n.obj
-        return None

nilmdb/server/interval.pyxdep

@@ -1 +0,0 @@
-rbtree.pxd

nilmdb/server/layout.pyx

@@ -1,209 +0,0 @@
-# cython: profile=False
-
-import time
-import sys
-import inspect
-import cStringIO
-import numpy as np
-
-cdef enum:
-    max_value_count = 64
-
-cimport cython
-cimport libc.stdlib
-cimport libc.stdio
-cimport libc.string
-
-class ParserError(Exception):
-    def __init__(self, line, message):
-        self.message = "line " + str(line) + ": " + message
-        Exception.__init__(self, self.message)
-
-class FormatterError(Exception):
-    pass
-
-class Layout:
-    """Represents a NILM database layout"""
-
-    def __init__(self, typestring):
-        """Initialize this Layout object to handle the specified
-        type string"""
-        try:
-            [ datatype, count ] = typestring.split("_")
-        except:
-            raise KeyError("invalid layout string")
-
-        try:
-            self.count = int(count)
-        except ValueError:
-            raise KeyError("invalid count")
-        if self.count < 1 or self.count > max_value_count:
-            raise KeyError("invalid count")
-
-        if datatype == 'uint16':
-            self.parse = self.parse_uint16
-            self.format = self.format_uint16
-        elif datatype == 'float32' or datatype == 'float64':
-            self.parse = self.parse_float64
-            self.format = self.format_float64
-        else:
-            raise KeyError("invalid type")
-
-        self.datatype = datatype
-
-    # Parsers
-    def parse_float64(self, char *text):
-        cdef int n
-        cdef double ts
-        # Return doubles even in float32 case, since they're going into
-        # a Python array which would upconvert to double anyway.
-        result = []
-        cdef char *end
-        ts = libc.stdlib.strtod(text, &end)
-        if end == text:
-            raise ValueError("bad timestamp")
-        result.append(ts)
-        for n in range(self.count):
-            text = end
-            result.append(libc.stdlib.strtod(text, &end))
-            if end == text:
-                raise ValueError("wrong number of values")
-        n = 0
-        while end[n] == ' ':
-            n += 1
-        if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
-            raise ValueError("extra data on line")
-        return (ts, result)
-
-    def parse_uint16(self, char *text):
-        cdef int n
-        cdef double ts
-        cdef int v
-        result = []
-        cdef char *end
-        ts = libc.stdlib.strtod(text, &end)
-        if end == text:
-            raise ValueError("bad timestamp")
-        result.append(ts)
-        for n in range(self.count):
-            text = end
-            v = libc.stdlib.strtol(text, &end, 10)
-            if v < 0 or v > 65535:
-                raise ValueError("value out of range")
-            result.append(v)
-            if end == text:
-                raise ValueError("wrong number of values")
-        n = 0
-        while end[n] == ' ':
-            n += 1
-        if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
-            raise ValueError("extra data on line")
-        return (ts, result)
-
-    # Formatters
-    def format_float64(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %f" % d[i+1]
-        return s + "\n"
-
-    def format_uint16(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %d" % d[i+1]
-        return s + "\n"
-
-# Get a layout by name
-def get_named(typestring):
-    try:
-        return Layout(typestring)
-    except KeyError:
-        compat = { "PrepData": "float32_8",
-                   "RawData": "uint16_6",
-                   "RawNotchedData": "uint16_9" }
-        return Layout(compat[typestring])
-
-class Parser(object):
-    """Object that parses and stores ASCII data for inclusion into the
-    database"""
-
-    def __init__(self, layout):
-        if issubclass(layout.__class__, Layout):
-            self.layout = layout
-        else:
-            try:
-                self.layout = get_named(layout)
-            except KeyError:
-                raise TypeError("unknown layout")
-
-        self.data = []
-        self.min_timestamp = None
-        self.max_timestamp = None
-
-    def parse(self, textdata):
-        """
-        Parse the data, provided as lines of text, using the current
-        layout, into an internal data structure suitable for a
-        pytables 'table.append(parser.data)'.
-        """
-        cdef double last_ts = 0, ts
-        cdef int n = 0, i
-        cdef char *line
-
-        indata = cStringIO.StringIO(textdata)
-        # Assume any parsing error is a real error.
-        # In the future we might want to skip completely empty lines,
-        # or partial lines right before EOF?
-        try:
-            self.data = []
-            for pyline in indata:
-                line = pyline
-                n += 1
-                if line[0] == '\#':
-                    continue
-                (ts, row) = self.layout.parse(line)
-                if ts <= last_ts:
-                    raise ValueError("timestamp is not "
-                                     "monotonically increasing")
-                last_ts = ts
-                self.data.append(row)
-        except (ValueError, IndexError, TypeError) as e:
-            raise ParserError(n, "error: " + e.message)
-
-        # Mark timestamp ranges
-        if len(self.data):
-            self.min_timestamp = self.data[0][0]
-            self.max_timestamp = self.data[-1][0]
-
-class Formatter(object):
-    """Object that formats database data into ASCII"""
-
-    def __init__(self, layout):
-        if issubclass(layout.__class__, Layout):
-            self.layout = layout
-        else:
-            try:
-                self.layout = get_named(layout)
-            except KeyError:
-                raise TypeError("unknown layout")
-
-    def format(self, data):
-        """
-        Format raw data from the database, using the current layout,
-        as lines of ACSII text.
-        """
-        text = cStringIO.StringIO()
-        try:
-            for row in data:
-                text.write(self.layout.format(row))
-        except (ValueError, IndexError, TypeError) as e:
-            raise FormatterError("formatting error: " + e.message)
-        return text.getvalue()

nilmdb/server/nilmdb.py

@@ -1,563 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""NilmDB
-
-Object that represents a NILM database file.
-
-Manages both the SQL database and the table storage backend.
-"""
-
-# Need absolute_import so that "import nilmdb" won't pull in
-# nilmdb.py, but will pull the parent nilmdb module instead.
-from __future__ import absolute_import
-import nilmdb
-from nilmdb.utils.printf import *
-from nilmdb.server.interval import (Interval, DBInterval,
-                                    IntervalSet, IntervalError)
-from nilmdb.server import bulkdata
-from nilmdb.server.errors import NilmDBError, StreamError, OverlapError
-
-import sqlite3
-import os
-import errno
-import bisect
-
-# Note about performance and transactions:
-#
-# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
-# takes about 125msec.  sqlite3 will commit transactions at 3 times:
-# 1: explicit con.commit()
-# 2: between a series of DML commands and non-DML commands, e.g.
-#    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
-# 3: at the end of an explicit transaction, e.g. "with self.con as con:"
-#
-# To speed up testing, or if this transaction speed becomes an issue,
-# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF.
-
-
-# Don't touch old entries -- just add new ones.
-_sql_schema_updates = {
-    0: """
-    -- All streams
-    CREATE TABLE streams(
-    	id INTEGER PRIMARY KEY,		-- stream ID
-        path TEXT UNIQUE NOT NULL,	-- path, e.g. '/newton/prep'
-        layout TEXT NOT NULL		-- layout name, e.g. float32_8
-    );
-
-    -- Individual timestamped ranges in those streams.
-    -- For a given start_time and end_time, this tells us that the
-    -- data is stored between start_pos and end_pos.
-    -- Times are stored as μs since Unix epoch
-    -- Positions are opaque: PyTables rows, file offsets, etc.
-    --
-    -- Note: end_pos points to the row _after_ end_time, so end_pos-1
-    -- is the last valid row.
-    CREATE TABLE ranges(
-        stream_id INTEGER NOT NULL,
-        start_time INTEGER NOT NULL,
-        end_time INTEGER NOT NULL,
-        start_pos INTEGER NOT NULL,
-        end_pos INTEGER NOT NULL
-    );
-    CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time);
-    """,
-
-    1: """
-    -- Generic dictionary-type metadata that can be associated with a stream
-    CREATE TABLE metadata(
-    	stream_id INTEGER NOT NULL,
-        key TEXT NOT NULL,
-        value TEXT
-    );
-    """,
-}
-
-@nilmdb.utils.must_close()
-class NilmDB(object):
-    verbose = 0
-
-    def __init__(self, basepath, sync=True, max_results=None,
-                 bulkdata_args=None):
-        if bulkdata_args is None:
-            bulkdata_args = {}
-
-        # set up path
-        self.basepath = os.path.abspath(basepath)
-
-        # Create the database path if it doesn't exist
-        try:
-            os.makedirs(self.basepath)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise IOError("can't create tree " + self.basepath)
-
-        # Our data goes inside it
-        self.data = bulkdata.BulkData(self.basepath, **bulkdata_args)
-
-        # SQLite database too
-        sqlfilename = os.path.join(self.basepath, "data.sql")
-        # We use check_same_thread = False, assuming that the rest
-        # of the code (e.g. Server) will be smart and not access this
-        # database from multiple threads simultaneously.  Otherwise
-        # false positives will occur when the database is only opened
-        # in one thread, and only accessed in another.
-        self.con = sqlite3.connect(sqlfilename, check_same_thread = False)
-        self._sql_schema_update()
-
-        # See big comment at top about the performance implications of this
-        if sync:
-            self.con.execute("PRAGMA synchronous=FULL")
-        else:
-            self.con.execute("PRAGMA synchronous=OFF")
-
-        # Approximate largest number of elements that we want to send
-        # in a single reply (for stream_intervals, stream_extract)
-        if max_results:
-            self.max_results = max_results
-        else:
-            self.max_results = 16384
-
-    def get_basepath(self):
-        return self.basepath
-
-    def close(self):
-        if self.con:
-            self.con.commit()
-            self.con.close()
-        self.data.close()
-
-    def _sql_schema_update(self):
-        cur = self.con.cursor()
-        version = cur.execute("PRAGMA user_version").fetchone()[0]
-        oldversion = version
-
-        while version in _sql_schema_updates:
-            cur.executescript(_sql_schema_updates[version])
-            version = version + 1
-            if self.verbose: # pragma: no cover
-                printf("Schema updated to %d\n", version)
-
-        if version != oldversion:
-            with self.con:
-                cur.execute("PRAGMA user_version = {v:d}".format(v=version))
-
-    def _check_user_times(self, start, end):
-        if start is None:
-            start = -1e12
-        if end is None:
-            end = 1e12
-        if start >= end:
-            raise NilmDBError("start must precede end")
-        return (start, end)
-
-    @nilmdb.utils.lru_cache(size = 16)
-    def _get_intervals(self, stream_id):
-        """
-        Return a mutable IntervalSet corresponding to the given stream ID.
-        """
-        iset = IntervalSet()
-        result = self.con.execute("SELECT start_time, end_time, "
-                                  "start_pos, end_pos "
-                                  "FROM ranges "
-                                  "WHERE stream_id=?", (stream_id,))
-        try:
-            for (start_time, end_time, start_pos, end_pos) in result:
-                iset += DBInterval(start_time, end_time,
-                                   start_time, end_time,
-                                   start_pos, end_pos)
-        except IntervalError: # pragma: no cover
-            raise NilmDBError("unexpected overlap in ranges table!")
-
-        return iset
-
-    def _sql_interval_insert(self, id, start, end, start_pos, end_pos):
-        """Helper that adds interval to the SQL database only"""
-        self.con.execute("INSERT INTO ranges "
-                         "(stream_id,start_time,end_time,start_pos,end_pos) "
-                         "VALUES (?,?,?,?,?)",
-                         (id, start, end, start_pos, end_pos))
-
-    def _sql_interval_delete(self, id, start, end, start_pos, end_pos):
-        """Helper that removes interval from the SQL database only"""
-        self.con.execute("DELETE FROM ranges WHERE "
-                         "stream_id=? AND start_time=? AND "
-                         "end_time=? AND start_pos=? AND end_pos=?",
-                         (id, start, end, start_pos, end_pos))
-
-    def _add_interval(self, stream_id, interval, start_pos, end_pos):
-        """
-        Add interval to the internal interval cache, and to the database.
-        Note: arguments must be ints (not numpy.int64, etc)
-        """
-        # Load this stream's intervals
-        iset = self._get_intervals(stream_id)
-
-        # Check for overlap
-        if iset.intersects(interval): # pragma: no cover (gets caught earlier)
-            raise NilmDBError("new interval overlaps existing data")
-
-        # Check for adjacency.  If there's a stream in the database
-        # that ends exactly when this one starts, and the database
-        # rows match up, we can make one interval that covers the
-        # time range [adjacent.start -> interval.end)
-        # and database rows [ adjacent.start_pos -> end_pos ].
-        # Only do this if the resulting interval isn't too large.
-        max_merged_rows = 8000 * 60 * 60 * 1.05 # 1.05 hours at 8 KHz
-        adjacent = iset.find_end(interval.start)
-        if (adjacent is not None and
-            start_pos == adjacent.db_endpos and
-            (end_pos - adjacent.db_startpos) < max_merged_rows):
-            # First delete the old one, both from our iset and the
-            # database
-            iset -= adjacent
-            self._sql_interval_delete(stream_id,
-                                      adjacent.db_start, adjacent.db_end,
-                                      adjacent.db_startpos, adjacent.db_endpos)
-
-            # Now update our interval so the fallthrough add is
-            # correct.
-            interval.start = adjacent.start
-            start_pos = adjacent.db_startpos
-
-        # Add the new interval to the iset
-        iset.iadd_nocheck(DBInterval(interval.start, interval.end,
-                                     interval.start, interval.end,
-                                     start_pos, end_pos))
-
-        # Insert into the database
-        self._sql_interval_insert(stream_id, interval.start, interval.end,
-                                  int(start_pos), int(end_pos))
-
-        self.con.commit()
-
-    def _remove_interval(self, stream_id, original, remove):
-        """
-        Remove an interval from the internal cache and the database.
-
-        stream_id: id of stream
-         original: original DBInterval; must be already present in DB
-        to_remove: DBInterval to remove; must be subset of 'original'
-        """
-        # Just return if we have nothing to remove
-        if remove.start == remove.end: # pragma: no cover
-            return
-
-        # Load this stream's intervals
-        iset = self._get_intervals(stream_id)
-
-        # Remove existing interval from the cached set and the database
-        iset -= original
-        self._sql_interval_delete(stream_id,
-                                  original.db_start, original.db_end,
-                                  original.db_startpos, original.db_endpos)
-
-        # Add back the intervals that would be left over if the
-        # requested interval is removed.  There may be two of them, if
-        # the removed piece was in the middle.
-        def add(iset, start, end, start_pos, end_pos):
-            iset += DBInterval(start, end, start, end, start_pos, end_pos)
-            self._sql_interval_insert(stream_id, start, end, start_pos, end_pos)
-
-        if original.start != remove.start:
-            # Interval before the removed region
-            add(iset, original.start, remove.start,
-                original.db_startpos, remove.db_startpos)
-
-        if original.end != remove.end:
-            # Interval after the removed region
-            add(iset, remove.end, original.end,
-                remove.db_endpos, original.db_endpos)
-
-        # Commit SQL changes
-        self.con.commit()
-
-        return
-
-    def stream_list(self, path = None, layout = None):
-        """Return list of [path, layout] lists of all streams
-        in the database.
-
-        If path is specified, include only streams with a path that
-        matches the given string.
-
-        If layout is specified, include only streams with a layout
-        that matches the given string.
-        """
-        where = "WHERE 1=1"
-        params = ()
-        if layout:
-            where += " AND layout=?"
-            params += (layout,)
-        if path:
-            where += " AND path=?"
-            params += (path,)
-        result = self.con.execute("SELECT path, layout "
-                                  "FROM streams " + where, params).fetchall()
-
-        return sorted(list(x) for x in result)
-
-    def stream_intervals(self, path, start = None, end = None):
-        """
-        Returns (intervals, restart) tuple.
-
-        intervals is a list of [start,end] timestamps of all intervals
-        that exist for path, between start and end.
-
-        restart, if nonzero, means that there were too many results to
-        return in a single request.  The data is complete from the
-        starting timestamp to the point at which it was truncated,
-        and a new request with a start time of 'restart' will fetch
-        the next block of data.
-        """
-        stream_id = self._stream_id(path)
-        intervals = self._get_intervals(stream_id)
-        (start, end) = self._check_user_times(start, end)
-        requested = Interval(start, end)
-        result = []
-        for n, i in enumerate(intervals.intersection(requested)):
-            if n >= self.max_results:
-                restart = i.start
-                break
-            result.append([i.start, i.end])
-        else:
-            restart = 0
-        return (result, restart)
-
-    def stream_create(self, path, layout_name):
-        """Create a new table in the database.
-
-        path: path to the data (e.g. '/newton/prep').
-        Paths must contain at least two elements, e.g.:
-           /newton/prep
-           /newton/raw
-           /newton/upstairs/prep
-           /newton/upstairs/raw
-
-        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
-        """
-        # Create the bulk storage.  Raises ValueError on error, which we
-        # pass along.
-        self.data.create(path, layout_name)
-
-        # Insert into SQL database once the bulk storage is happy
-        with self.con as con:
-            con.execute("INSERT INTO streams (path, layout) VALUES (?,?)",
-                        (path, layout_name))
-
-    def _stream_id(self, path):
-        """Return unique stream ID"""
-        result = self.con.execute("SELECT id FROM streams WHERE path=?",
-                                  (path,)).fetchone()
-        if result is None:
-            raise StreamError("No stream at path " + path)
-        return result[0]
-
-    def stream_set_metadata(self, path, data):
-        """Set stream metadata from a dictionary, e.g.
-           { description = 'Downstairs lighting',
-             v_scaling = 123.45 }
-           This replaces all existing metadata.
-           """
-        stream_id = self._stream_id(path)
-        with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
-            for key in data:
-                if data[key] != '':
-                    con.execute("INSERT INTO metadata VALUES (?, ?, ?)",
-                                (stream_id, key, data[key]))
-
-    def stream_get_metadata(self, path):
-        """Return stream metadata as a dictionary."""
-        stream_id = self._stream_id(path)
-        result = self.con.execute("SELECT metadata.key, metadata.value "
-                                  "FROM metadata "
-                                  "WHERE metadata.stream_id=?", (stream_id,))
-        data = {}
-        for (key, value) in result:
-            data[key] = value
-        return data
-
-    def stream_update_metadata(self, path, newdata):
-        """Update stream metadata from a dictionary"""
-        data = self.stream_get_metadata(path)
-        data.update(newdata)
-        self.stream_set_metadata(path, data)
-
-    def stream_destroy(self, path):
-        """Fully remove a table and all of its data from the database.
-        No way to undo it!  Metadata is removed."""
-        stream_id = self._stream_id(path)
-
-        # Delete the cached interval data (if it was cached)
-        self._get_intervals.cache_remove(self, stream_id)
-
-        # Delete the data
-        self.data.destroy(path)
-
-        # Delete metadata, stream, intervals
-        with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
-
-    def stream_insert(self, path, start, end, data):
-        """Insert new data into the database.
-           path: Path at which to add the data
-           start: Starting timestamp
-           end: Ending timestamp
-           data: Rows of data, to be passed to bulkdata table.append
-                 method.  E.g. nilmdb.layout.Parser.data
-           """
-        # First check for basic overlap using timestamp info given.
-        stream_id = self._stream_id(path)
-        iset = self._get_intervals(stream_id)
-        interval = Interval(start, end)
-        if iset.intersects(interval):
-            raise OverlapError("new data overlaps existing data at range: "
-                               + str(iset & interval))
-
-        # Insert the data
-        table = self.data.getnode(path)
-        row_start = table.nrows
-        table.append(data)
-        row_end = table.nrows
-
-        # Insert the record into the sql database.
-        self._add_interval(stream_id, interval, row_start, row_end)
-
-        # And that's all
-        return
-
-    def _find_start(self, table, dbinterval):
-        """
-        Given a DBInterval, find the row in the database that
-        corresponds to the start time.  Return the first database
-        position with a timestamp (first element) greater than or
-        equal to 'start'.
-        """
-        # Optimization for the common case where an interval wasn't truncated
-        if dbinterval.start == dbinterval.db_start:
-            return dbinterval.db_startpos
-        return bisect.bisect_left(bulkdata.TimestampOnlyTable(table),
-                                  dbinterval.start,
-                                  dbinterval.db_startpos,
-                                  dbinterval.db_endpos)
-
-    def _find_end(self, table, dbinterval):
-        """
-        Given a DBInterval, find the row in the database that follows
-        the end time.  Return the first database position after the
-        row with timestamp (first element) greater than or equal
-        to 'end'.
-        """
-        # Optimization for the common case where an interval wasn't truncated
-        if dbinterval.end == dbinterval.db_end:
-            return dbinterval.db_endpos
-        # Note that we still use bisect_left here, because we don't
-        # want to include the given timestamp in the results.  This is
-        # so a queries like 1:00 -> 2:00 and 2:00 -> 3:00 return
-        # non-overlapping data.
-        return bisect.bisect_left(bulkdata.TimestampOnlyTable(table),
-                                  dbinterval.end,
-                                  dbinterval.db_startpos,
-                                  dbinterval.db_endpos)
-
-    def stream_extract(self, path, start = None, end = None, count = False):
-        """
-        Returns (data, restart) tuple.
-
-        data is a list of raw data from the database, suitable for
-        passing to e.g. nilmdb.layout.Formatter to translate into
-        textual form.
-
-        restart, if nonzero, means that there were too many results to
-        return in a single request.  The data is complete from the
-        starting timestamp to the point at which it was truncated,
-        and a new request with a start time of 'restart' will fetch
-        the next block of data.
-
-        count, if true, means to not return raw data, but just the count
-        of rows that would have been returned.  This is much faster
-        than actually fetching the data.  It is not limited by
-        max_results.
-        """
-        stream_id = self._stream_id(path)
-        table = self.data.getnode(path)
-        intervals = self._get_intervals(stream_id)
-        (start, end) = self._check_user_times(start, end)
-        requested = Interval(start, end)
-        result = []
-        matched = 0
-        remaining = self.max_results
-        restart = 0
-        for interval in intervals.intersection(requested):
-            # Reading single rows from the table is too slow, so
-            # we use two bisections to find both the starting and
-            # ending row for this particular interval, then
-            # read the entire range as one slice.
-            row_start = self._find_start(table, interval)
-            row_end = self._find_end(table, interval)
-
-            if count:
-                matched += row_end - row_start
-                continue
-
-            # Shorten it if we'll hit the maximum number of results
-            row_max = row_start + remaining
-            if row_max < row_end:
-                row_end = row_max
-                restart = table[row_max][0]
-
-            # Gather these results up
-            result.extend(table[row_start:row_end])
-
-            # Count them
-            remaining -= row_end - row_start
-
-            if restart:
-                break
-
-        if count:
-            return matched
-        return (result, restart)
-
-    def stream_remove(self, path, start = None, end = None):
-        """
-        Remove data from the specified time interval within a stream.
-        Removes all data in the interval [start, end), and intervals
-        are truncated or split appropriately.  Returns the number of
-        data points removed.
-        """
-        stream_id = self._stream_id(path)
-        table = self.data.getnode(path)
-        intervals = self._get_intervals(stream_id)
-        (start, end) = self._check_user_times(start, end)
-        to_remove = Interval(start, end)
-        removed = 0
-
-        # Can't remove intervals from within the iterator, so we need to
-        # remember what's currently in the intersection now.
-        all_candidates = list(intervals.intersection(to_remove, orig = True))
-
-        for (dbint, orig) in all_candidates:
-            # Find row start and end
-            row_start = self._find_start(table, dbint)
-            row_end = self._find_end(table, dbint)
-
-            # Adjust the DBInterval to match the newly found ends
-            dbint.db_start = dbint.start
-            dbint.db_end = dbint.end
-            dbint.db_startpos = row_start
-            dbint.db_endpos = row_end
-
-            # Remove interval from the database
-            self._remove_interval(stream_id, orig, dbint)
-
-            # Remove data from the underlying table storage
-            table.remove(row_start, row_end)
-
-            # Count how many were removed
-            removed += row_end - row_start
-
-        return removed

nilmdb/server/rbtree.pxd

@@ -1,23 +0,0 @@
-cdef class RBNode:
-    cdef public object obj
-    cdef public double start, end
-    cdef public int red
-    cdef public RBNode left, right, parent
-
-cdef class RBTree:
-    cdef public RBNode nil, root
-
-    cpdef getroot(RBTree self)
-    cdef void __rotate_left(RBTree self, RBNode x)
-    cdef void __rotate_right(RBTree self, RBNode y)
-    cdef RBNode __successor(RBTree self, RBNode x)
-    cpdef RBNode successor(RBTree self, RBNode x)
-    cdef RBNode __predecessor(RBTree self, RBNode x)
-    cpdef RBNode predecessor(RBTree self, RBNode x)
-    cpdef insert(RBTree self, RBNode z)
-    cdef void __insert_fixup(RBTree self, RBNode x)
-    cpdef delete(RBTree self, RBNode z)
-    cdef inline void __delete_fixup(RBTree self, RBNode x)
-    cpdef RBNode find(RBTree self, double start, double end)
-    cpdef RBNode find_left_end(RBTree self, double t)
-    cpdef RBNode find_right_start(RBTree self, double t)

nilmdb/server/rbtree.pyx

@@ -1,377 +0,0 @@
-# cython: profile=False
-# cython: cdivision=True
-
-"""
-Jim Paris <jim@jtan.com>
-
-Red-black tree, where keys are stored as start/end timestamps.
-This is a basic interval tree that holds half-open intervals:
-  [start, end)
-Intervals must not overlap.  Fixing that would involve making this
-into an augmented interval tree as described in CLRS 14.3.
-
-Code that assumes non-overlapping intervals is marked with the
-string 'non-overlapping'.
-"""
-
-import sys
-cimport rbtree
-
-cdef class RBNode:
-    """One node of the Red/Black tree, containing a key (start, end)
-    and value (obj)"""
-    def __init__(self, double start, double end, object obj = None):
-        self.obj = obj
-        self.start = start
-        self.end = end
-        self.red = False
-        self.left = None
-        self.right = None
-
-    def __str__(self):
-        if self.red:
-            color = "R"
-        else:
-            color = "B"
-        if self.start == sys.float_info.min:
-            return "[node nil]"
-        return ("[node ("
-                + str(self.obj) + ") "
-                + str(self.start) + " -> " + str(self.end) + " "
-                + color + "]")
-
-cdef class RBTree:
-    """Red/Black tree"""
-
-    # Init
-    def __init__(self):
-        self.nil = RBNode(start = sys.float_info.min,
-                          end = sys.float_info.min)
-        self.nil.left = self.nil
-        self.nil.right = self.nil
-        self.nil.parent = self.nil
-
-        self.root = RBNode(start = sys.float_info.max,
-                           end = sys.float_info.max)
-        self.root.left = self.nil
-        self.root.right = self.nil
-        self.root.parent = self.nil
-
-    # We have a dummy root node to simplify operations, so from an
-    # external point of view, its left child is the real root.
-    cpdef getroot(self):
-        return self.root.left
-
-    # Rotations and basic operations
-    cdef void __rotate_left(self, RBNode x):
-        """Rotate left:
-        #   x           y
-        #  / \   -->   / \
-        # z   y       x   w
-        #    / \     / \
-        #   v   w   z   v
-        """
-        cdef RBNode y = x.right
-        x.right = y.left
-        if y.left is not self.nil:
-            y.left.parent = x
-        y.parent = x.parent
-        if x is x.parent.left:
-            x.parent.left = y
-        else:
-            x.parent.right = y
-        y.left = x
-        x.parent = y
-
-    cdef void __rotate_right(self, RBNode y):
-        """Rotate right:
-        #     y           x
-        #    / \   -->   / \
-        #   x   w       z   y
-        #  / \             / \
-        # z   v           v   w
-        """
-        cdef RBNode x = y.left
-        y.left = x.right
-        if x.right is not self.nil:
-            x.right.parent = y
-        x.parent = y.parent
-        if y is y.parent.left:
-            y.parent.left = x
-        else:
-            y.parent.right = x
-        x.right = y
-        y.parent = x
-
-    cdef RBNode __successor(self, RBNode x):
-        """Returns the successor of RBNode x"""
-        cdef RBNode y = x.right
-        if y is not self.nil:
-            while y.left is not self.nil:
-                y = y.left
-        else:
-            y = x.parent
-            while x is y.right:
-                x = y
-                y = y.parent
-            if y is self.root:
-                return self.nil
-        return y
-    cpdef RBNode successor(self, RBNode x):
-        """Returns the successor of RBNode x, or None"""
-        cdef RBNode y = self.__successor(x)
-        return y if y is not self.nil else None
-
-    cdef RBNode __predecessor(self, RBNode x):
-        """Returns the predecessor of RBNode x"""
-        cdef RBNode y = x.left
-        if y is not self.nil:
-            while y.right is not self.nil:
-                y = y.right
-        else:
-            y = x.parent
-            while x is y.left:
-                if y is self.root:
-                    y = self.nil
-                    break
-                x = y
-                y = y.parent
-        return y
-    cpdef RBNode predecessor(self, RBNode x):
-        """Returns the predecessor of RBNode x, or None"""
-        cdef RBNode y = self.__predecessor(x)
-        return y if y is not self.nil else None
-
-    # Insertion
-    cpdef insert(self, RBNode z):
-        """Insert RBNode z into RBTree and rebalance as necessary"""
-        z.left = self.nil
-        z.right = self.nil
-        cdef RBNode y = self.root
-        cdef RBNode x = self.root.left
-        while x is not self.nil:
-            y = x
-            if (x.start > z.start or (x.start == z.start and x.end > z.end)):
-                x = x.left
-            else:
-                x = x.right
-        z.parent = y
-        if (y is self.root or
-            (y.start > z.start or (y.start == z.start and y.end > z.end))):
-            y.left = z
-        else:
-            y.right = z
-        # relabel/rebalance
-        self.__insert_fixup(z)
-
-    cdef void __insert_fixup(self, RBNode x):
-        """Rebalance/fix RBTree after a simple insertion of RBNode x"""
-        x.red = True
-        while x.parent.red:
-            if x.parent is x.parent.parent.left:
-                y = x.parent.parent.right
-                if y.red:
-                    x.parent.red = False
-                    y.red = False
-                    x.parent.parent.red = True
-                    x = x.parent.parent
-                else:
-                    if x is x.parent.right:
-                        x = x.parent
-                        self.__rotate_left(x)
-                    x.parent.red = False
-                    x.parent.parent.red = True
-                    self.__rotate_right(x.parent.parent)
-            else: # same as above, left/right switched
-                y = x.parent.parent.left
-                if y.red:
-                    x.parent.red = False
-                    y.red = False
-                    x.parent.parent.red = True
-                    x = x.parent.parent
-                else:
-                    if x is x.parent.left:
-                        x = x.parent
-                        self.__rotate_right(x)
-                    x.parent.red = False
-                    x.parent.parent.red = True
-                    self.__rotate_left(x.parent.parent)
-        self.root.left.red = False
-
-    # Deletion
-    cpdef delete(self, RBNode z):
-        if z.left is None or z.right is None:
-            raise AttributeError("you can only delete a node object "
-                                 + "from the tree; use find() to get one")
-        cdef RBNode x, y
-        if z.left is self.nil or z.right is self.nil:
-            y = z
-        else:
-            y = self.__successor(z)
-        if y.left is self.nil:
-            x = y.right
-        else:
-            x = y.left
-        x.parent = y.parent
-        if x.parent is self.root:
-            self.root.left = x
-        else:
-            if y is y.parent.left:
-                y.parent.left = x
-            else:
-                y.parent.right = x
-        if y is not z:
-            # y is the node to splice out, x is its child
-            y.left = z.left
-            y.right = z.right
-            y.parent = z.parent
-            z.left.parent = y
-            z.right.parent = y
-            if z is z.parent.left:
-                z.parent.left = y
-            else:
-                z.parent.right = y
-            if not y.red:
-                y.red = z.red
-                self.__delete_fixup(x)
-            else:
-                y.red = z.red
-        else:
-            if not y.red:
-                self.__delete_fixup(x)
-
-    cdef void __delete_fixup(self, RBNode x):
-        """Rebalance/fix RBTree after a deletion.  RBNode x is the
-        child of the spliced out node."""
-        cdef RBNode rootLeft = self.root.left
-        while not x.red and x is not rootLeft:
-            if x is x.parent.left:
-                w = x.parent.right
-                if w.red:
-                    w.red = False
-                    x.parent.red = True
-                    self.__rotate_left(x.parent)
-                    w = x.parent.right
-                if not w.right.red and not w.left.red:
-                    w.red = True
-                    x = x.parent
-                else:
-                    if not w.right.red:
-                        w.left.red = False
-                        w.red = True
-                        self.__rotate_right(w)
-                        w = x.parent.right
-                    w.red = x.parent.red
-                    x.parent.red = False
-                    w.right.red = False
-                    self.__rotate_left(x.parent)
-                    x = rootLeft # exit loop
-            else: # same as above, left/right switched
-                w = x.parent.left
-                if w.red:
-                    w.red = False
-                    x.parent.red = True
-                    self.__rotate_right(x.parent)
-                    w = x.parent.left
-                if not w.left.red and not w.right.red:
-                    w.red = True
-                    x = x.parent
-                else:
-                    if not w.left.red:
-                        w.right.red = False
-                        w.red = True
-                        self.__rotate_left(w)
-                        w = x.parent.left
-                    w.red = x.parent.red
-                    x.parent.red = False
-                    w.left.red = False
-                    self.__rotate_right(x.parent)
-                    x = rootLeft # exit loop
-        x.red = False
-
-    # Walking, searching
-    def __iter__(self):
-        return self.inorder()
-
-    def inorder(self, RBNode x = None):
-        """Generator that performs an inorder walk for the tree
-        rooted at RBNode x"""
-        if x is None:
-            x = self.getroot()
-        while x.left is not self.nil:
-            x = x.left
-        while x is not self.nil:
-            yield x
-            x = self.__successor(x)
-
-    cpdef RBNode find(self, double start, double end):
-        """Return the node with exactly the given start and end."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if start < x.start:
-                x = x.left
-            elif start == x.start:
-                if end == x.end:
-                    break # found it
-                elif end < x.end:
-                    x = x.left
-                else:
-                    x = x.right
-            else:
-                x = x.right
-        return x if x is not self.nil else None
-
-    cpdef RBNode find_left_end(self, double t):
-        """Find the leftmode node with end >= t.  With non-overlapping
-        intervals, this is the first node that might overlap time t.
-
-        Note that this relies on non-overlapping intervals, since
-        it assumes that we can use the endpoints to traverse the
-        tree even though it was created using the start points."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if t < x.end:
-                if x.left is self.nil:
-                    break
-                x = x.left
-            elif t == x.end:
-                break
-            else:
-                if x.right is self.nil:
-                    x = self.__successor(x)
-                    break
-                x = x.right
-        return x if x is not self.nil else None
-
-    cpdef RBNode find_right_start(self, double t):
-        """Find the rightmode node with start <= t.  With non-overlapping
-        intervals, this is the last node that might overlap time t."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if t < x.start:
-                if x.left is self.nil:
-                    x = self.__predecessor(x)
-                    break
-                x = x.left
-            elif t == x.start:
-                break
-            else:
-                if x.right is self.nil:
-                    break
-                x = x.right
-        return x if x is not self.nil else None
-
-    # Intersections
-    def intersect(self, double start, double end):
-        """Generator that returns nodes that overlap the given
-        (start,end) range.  Assumes non-overlapping intervals."""
-        # Start with the leftmode node that ends after start
-        cdef RBNode n = self.find_left_end(start)
-        while n is not None:
-            if n.start >= end:
-                # this node starts after the requested end; we're done
-                break
-            if start < n.end:
-                # this node overlaps our requested area
-                yield n
-            n = self.successor(n)

nilmdb/server/rbtree.pyxdep

@@ -1 +0,0 @@
-rbtree.pxd

nilmdb/server/server.py

@@ -1,502 +0,0 @@
-"""CherryPy-based server for accessing NILM database via HTTP"""
-
-# Need absolute_import so that "import nilmdb" won't pull in
-# nilmdb.py, but will pull the nilmdb module instead.
-from __future__ import absolute_import
-import nilmdb
-from nilmdb.utils.printf import *
-from nilmdb.server.errors import NilmDBError
-
-import cherrypy
-import sys
-import os
-import simplejson as json
-import decorator
-import traceback
-import psutil
-
-try:
-    cherrypy.tools.json_out
-except: # pragma: no cover
-    sys.stderr.write("Cherrypy 3.2+ required\n")
-    sys.exit(1)
-
-class NilmApp(object):
-    def __init__(self, db):
-        self.db = db
-
-# Decorators
-def chunked_response(func):
-    """Decorator to enable chunked responses."""
-    # Set this to False to get better tracebacks from some requests
-    # (/stream/extract, /stream/intervals).
-    func._cp_config = { 'response.stream': True }
-    return func
-
-def response_type(content_type):
-    """Return a decorator-generating function that sets the
-    response type to the specified string."""
-    def wrapper(func, *args, **kwargs):
-        cherrypy.response.headers['Content-Type'] = content_type
-        return func(*args, **kwargs)
-    return decorator.decorator(wrapper)
-
-@decorator.decorator
-def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
-    """Decorator to work around CherryPy bug #1200 in a response
-    generator.
-
-    Even if chunked responses are disabled, LookupError or
-    UnicodeError exceptions may still be swallowed by CherryPy due to
-    bug #1200.  This throws them as generic Exceptions instead so that
-    they make it through.
-    """
-    try:
-        for val in func(*args, **kwargs):
-            yield val
-    except (LookupError, UnicodeError):
-        raise Exception("bug workaround; real exception is:\n" +
-                        traceback.format_exc())
-
-def exception_to_httperror(*expected):
-    """Return a decorator-generating function that catches expected
-    errors and throws a HTTPError describing it instead.
-
-        @exception_to_httperror(NilmDBError, ValueError)
-        def foo():
-            pass
-    """
-    def wrapper(func, *args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except expected as e:
-            message = sprintf("%s", str(e))
-            raise cherrypy.HTTPError("400 Bad Request", message)
-    # We need to preserve the function's argspecs for CherryPy to
-    # handle argument errors correctly.  Decorator.decorator takes
-    # care of that.
-    return decorator.decorator(wrapper)
-
-# CherryPy apps
-class Root(NilmApp):
-    """Root application for NILM database"""
-
-    def __init__(self, db):
-        super(Root, self).__init__(db)
-
-    # /
-    @cherrypy.expose
-    def index(self):
-        raise cherrypy.NotFound()
-
-    # /favicon.ico
-    @cherrypy.expose
-    def favicon_ico(self):
-        raise cherrypy.NotFound()
-
-    # /version
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def version(self):
-        return nilmdb.__version__
-
-    # /dbinfo
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def dbinfo(self):
-        """Return a dictionary with the database path,
-        size of the database in bytes, and free disk space in bytes"""
-        path = self.db.get_basepath()
-        return { "path": path,
-                 "size": nilmdb.utils.du(path),
-                 "free": psutil.disk_usage(path).free }
-
-class Stream(NilmApp):
-    """Stream-specific operations"""
-
-    # /stream/list
-    # /stream/list?layout=PrepData
-    # /stream/list?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def list(self, path = None, layout = None):
-        """List all streams in the database.  With optional path or
-        layout parameter, just list streams that match the given path
-        or layout"""
-        return self.db.stream_list(path, layout)
-
-    # /stream/create?path=/newton/prep&layout=PrepData
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, ValueError)
-    def create(self, path, layout):
-        """Create a new stream in the database.  Provide path
-        and one of the nilmdb.layout.layouts keys.
-        """
-        return self.db.stream_create(path, layout)
-
-    # /stream/destroy?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError)
-    def destroy(self, path):
-        """Delete a stream and its associated data."""
-        return self.db.stream_destroy(path)
-
-    # /stream/get_metadata?path=/newton/prep
-    # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def get_metadata(self, path, key=None):
-        """Get metadata for the named stream.  If optional
-        key parameters are specified, only return metadata
-        matching the given keys."""
-        try:
-            data = self.db.stream_get_metadata(path)
-        except nilmdb.server.nilmdb.StreamError as e:
-            raise cherrypy.HTTPError("404 Not Found", e.message)
-        if key is None:  # If no keys specified, return them all
-            key = data.keys()
-        elif not isinstance(key, list):
-            key = [ key ]
-        result = {}
-        for k in key:
-            if k in data:
-                result[k] = data[k]
-            else: # Return "None" for keys with no matching value
-                result[k] = None
-        return result
-
-    # /stream/set_metadata?path=/newton/prep&data=<json>
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError, TypeError)
-    def set_metadata(self, path, data):
-        """Set metadata for the named stream, replacing any
-        existing metadata.  Data should be a json-encoded
-        dictionary"""
-        data_dict = json.loads(data)
-        self.db.stream_set_metadata(path, data_dict)
-
-    # /stream/update_metadata?path=/newton/prep&data=<json>
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError, TypeError)
-    def update_metadata(self, path, data):
-        """Update metadata for the named stream.  Data
-        should be a json-encoded dictionary"""
-        data_dict = json.loads(data)
-        self.db.stream_update_metadata(path, data_dict)
-
-    # /stream/insert?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    #@cherrypy.tools.disable_prb()
-    def insert(self, path, start, end):
-        """
-        Insert new data into the database.  Provide textual data
-        (matching the path's layout) as a HTTP PUT.
-        """
-        # Important that we always read the input before throwing any
-        # errors, to keep lengths happy for persistent connections.
-        # However, CherryPy 3.2.2 has a bug where this fails for GET
-        # requests, so catch that. (issue #1134)
-        try:
-            body = cherrypy.request.body.read()
-        except TypeError:
-            raise cherrypy.HTTPError("400 Bad Request", "No request body")
-
-        # Check path and get layout
-        streams = self.db.stream_list(path = path)
-        if len(streams) != 1:
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
-
-        # Parse the input data
-        try:
-            parser = nilmdb.server.layout.Parser(layout)
-            parser.parse(body)
-        except nilmdb.server.layout.ParserError as e:
-            raise cherrypy.HTTPError("400 Bad Request",
-                                     "error parsing input data: " +
-                                     e.message)
-
-        # Check limits
-        start = float(start)
-        end = float(end)
-        if start >= end:
-            raise cherrypy.HTTPError("400 Bad Request",
-                                     "start must precede end")
-        if parser.min_timestamp is not None and parser.min_timestamp < start:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.min_timestamp) +
-                                     " < start time " + repr(start))
-        if parser.max_timestamp is not None and parser.max_timestamp >= end:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.max_timestamp) +
-                                     " >= end time " + repr(end))
-
-        # Now do the nilmdb insert, passing it the parser full of data.
-        try:
-            self.db.stream_insert(path, start, end, parser.data)
-        except NilmDBError as e:
-            raise cherrypy.HTTPError("400 Bad Request", e.message)
-
-        # Done
-        return
-
-    # /stream/remove?path=/newton/prep
-    # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError)
-    def remove(self, path, start = None, end = None):
-        """
-        Remove data from the backend database.  Removes all data in
-        the interval [start, end).  Returns the number of data points
-        removed.
-        """
-        if start is not None:
-            start = float(start)
-        if end is not None:
-            end = float(end)
-        if start is not None and end is not None:
-            if start >= end:
-                raise cherrypy.HTTPError("400 Bad Request",
-                                         "start must precede end")
-        return self.db.stream_remove(path, start, end)
-
-    # /stream/intervals?path=/newton/prep
-    # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    @cherrypy.expose
-    @chunked_response
-    @response_type("text/plain")
-    def intervals(self, path, start = None, end = None):
-        """
-        Get intervals from backend database.  Streams the resulting
-        intervals as JSON strings separated by newlines.  This may
-        make multiple requests to the nilmdb backend to avoid causing
-        it to block for too long.
-
-        Note that the response type is set to 'text/plain' even
-        though we're sending back JSON; this is because we're not
-        really returning a single JSON object.
-        """
-        if start is not None:
-            start = float(start)
-        if end is not None:
-            end = float(end)
-
-        if start is not None and end is not None:
-            if start >= end:
-                raise cherrypy.HTTPError("400 Bad Request",
-                                         "start must precede end")
-
-        streams = self.db.stream_list(path = path)
-        if len(streams) != 1:
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-
-        @workaround_cp_bug_1200
-        def content(start, end):
-            # Note: disable chunked responses to see tracebacks from here.
-            while True:
-                (intervals, restart) = self.db.stream_intervals(path, start, end)
-                response = ''.join([ json.dumps(i) + "\n" for i in intervals ])
-                yield response
-                if restart == 0:
-                    break
-                start = restart
-        return content(start, end)
-
-    # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    @cherrypy.expose
-    @chunked_response
-    @response_type("text/plain")
-    def extract(self, path, start = None, end = None, count = False):
-        """
-        Extract data from backend database.  Streams the resulting
-        entries as ASCII text lines separated by newlines.  This may
-        make multiple requests to the nilmdb backend to avoid causing
-        it to block for too long.
-
-        Add count=True to return a count rather than actual data.
-        """
-        if start is not None:
-            start = float(start)
-        if end is not None:
-            end = float(end)
-
-        # Check parameters
-        if start is not None and end is not None:
-            if start >= end:
-                raise cherrypy.HTTPError("400 Bad Request",
-                                         "start must precede end")
-
-        # Check path and get layout
-        streams = self.db.stream_list(path = path)
-        if len(streams) != 1:
-            raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
-
-        # Get formatter
-        formatter = nilmdb.server.layout.Formatter(layout)
-
-        @workaround_cp_bug_1200
-        def content(start, end, count):
-            # Note: disable chunked responses to see tracebacks from here.
-            if count:
-                matched = self.db.stream_extract(path, start, end, count)
-                yield sprintf("%d\n", matched)
-                return
-
-            while True:
-                (data, restart) = self.db.stream_extract(path, start, end)
-
-                # Format the data and yield it
-                yield formatter.format(data)
-
-                if restart == 0:
-                    return
-                start = restart
-        return content(start, end, count)
-
-class Exiter(object):
-    """App that exits the server, for testing"""
-    @cherrypy.expose
-    def index(self):
-        cherrypy.response.headers['Content-Type'] = 'text/plain'
-        def content():
-            yield 'Exiting by request'
-            raise SystemExit
-        return content()
-    index._cp_config = { 'response.stream': True }
-
-class Server(object):
-    def __init__(self, db, host = '127.0.0.1', port = 8080,
-                 stoppable = False,       # whether /exit URL exists
-                 embedded = True,         # hide diagnostics and output, etc
-                 fast_shutdown = False,   # don't wait for clients to disconn.
-                 force_traceback = False  # include traceback in all errors
-                 ):
-        # Save server version, just for verification during tests
-        self.version = nilmdb.__version__
-
-        # Need to wrap DB object in a serializer because we'll call
-        # into it from separate threads.
-        self.embedded = embedded
-        self.db = nilmdb.utils.Serializer(db)
-
-        # Build up global server configuration
-        cherrypy.config.update({
-            'server.socket_host': host,
-            'server.socket_port': port,
-            'engine.autoreload_on': False,
-            'server.max_request_body_size': 4*1024*1024,
-            })
-        if self.embedded:
-            cherrypy.config.update({ 'environment': 'embedded' })
-
-        # Build up application specific configuration
-        app_config = {}
-        app_config.update({
-            'error_page.default': self.json_error_page,
-            })
-
-        # Send a permissive Access-Control-Allow-Origin (CORS) header
-        # with all responses so that browsers can send cross-domain
-        # requests to this server.
-        app_config.update({ 'response.headers.Access-Control-Allow-Origin':
-                            '*' })
-
-        # Send tracebacks in error responses.  They're hidden by the
-        # error_page function for client errors (code 400-499).
-        app_config.update({ 'request.show_tracebacks' : True })
-        self.force_traceback = force_traceback
-
-        # Patch CherryPy error handler to never pad out error messages.
-        # This isn't necessary, but then again, neither is padding the
-        # error messages.
-        cherrypy._cperror._ie_friendly_error_sizes = {}
-
-        # Build up the application and mount it
-        root = Root(self.db)
-        root.stream = Stream(self.db)
-        if stoppable:
-            root.exit = Exiter()
-        cherrypy.tree.apps = {}
-        cherrypy.tree.mount(root, "/", config = { "/" : app_config })
-
-        # Shutdowns normally wait for clients to disconnect.  To speed
-        # up tests, set fast_shutdown = True
-        if fast_shutdown:
-            # Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
-            cherrypy.server.shutdown_timeout = 0.01
-        else:
-            cherrypy.server.shutdown_timeout = 5
-
-    def json_error_page(self, status, message, traceback, version):
-        """Return a custom error page in JSON so the client can parse it"""
-        errordata = { "status" : status,
-                      "message" : message,
-                      "traceback" : traceback }
-        # Don't send a traceback if the error was 400-499 (client's fault)
-        try:
-            code = int(status.split()[0])
-            if not self.force_traceback:
-                if code >= 400 and code <= 499:
-                    errordata["traceback"] = ""
-        except Exception: # pragma: no cover
-            pass
-        # Override the response type, which was previously set to text/html
-        cherrypy.serving.response.headers['Content-Type'] = (
-            "application/json;charset=utf-8" )
-        # Undo the HTML escaping that cherrypy's get_error_page function applies
-        # (cherrypy issue 1135)
-        for k, v in errordata.iteritems():
-            v = v.replace("&lt;","<")
-            v = v.replace("&gt;",">")
-            v = v.replace("&amp;","&")
-            errordata[k] = v
-        return json.dumps(errordata, separators=(',',':'))
-
-    def start(self, blocking = False, event = None):
-
-        if not self.embedded: # pragma: no cover
-            # Handle signals nicely
-            if hasattr(cherrypy.engine, "signal_handler"):
-                cherrypy.engine.signal_handler.subscribe()
-            if hasattr(cherrypy.engine, "console_control_handler"):
-                cherrypy.engine.console_control_handler.subscribe()
-
-        # Cherrypy stupidly calls os._exit(70) when it can't bind the
-        # port.  At least try to print a reasonable error and continue
-        # in this case, rather than just dying silently (as we would
-        # otherwise do in embedded mode)
-        real_exit = os._exit
-        def fake_exit(code): # pragma: no cover
-            if code == os.EX_SOFTWARE:
-                fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
-            else:
-                real_exit(code)
-        os._exit = fake_exit
-        cherrypy.engine.start()
-        os._exit = real_exit
-
-        # Signal that the engine has started successfully
-        if event is not None:
-            event.set()
-
-        if blocking:
-            try:
-                cherrypy.engine.wait(cherrypy.engine.states.EXITING,
-                                     interval = 0.1, channel = 'main')
-            except (KeyboardInterrupt, IOError): # pragma: no cover
-                cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
-                cherrypy.engine.exit()
-            except SystemExit: # pragma: no cover
-                cherrypy.engine.log('SystemExit raised: shutting down bus')
-                cherrypy.engine.exit()
-                raise
-
-    def stop(self):
-        cherrypy.engine.exit()

nilmdb/test_fileinterval.py

@@ -0,0 +1,46 @@
+from nilmdb import Interval, IntervalSet, IntervalError, FileInterval
+from datetime import datetime
+from nose.tools import assert_raises
+
+from test_interval import iset
+
+def fiset(string):
+    """Like iset, but builds with FileIntervals instead of Intervals"""
+    iset = IntervalSet()
+    for i, c in enumerate(string):
+        day = datetime.strptime("{0:04d}".format(i+2000), "%Y")
+        if (c == "["):
+            start = day
+        elif (c == "|"):
+            iset += FileInterval(start, day, "test.dat")
+            start = day
+        elif (c == "]"):
+            iset += FileInterval(start, day, "test.dat")
+            del start
+    return iset
+
+def test_fileinterval_vs_interval():
+    """Test FileInterval/Interval inheritance"""
+
+    i = iset("[--]")
+    f = fiset("[--]")
+
+    # check types
+    assert(isinstance(i[0], Interval))
+    assert(not isinstance(i[0], FileInterval))
+    assert(isinstance(f[0], Interval))
+    assert(isinstance(f[0], FileInterval))
+
+    # when doing an intersection, result should be a subset of the first arg
+    u = (i & f)
+    assert(isinstance(u[0], Interval))
+    assert(not isinstance(u[0], FileInterval))
+    u = (f & i)
+    assert(isinstance(u[0], Interval))
+    assert(isinstance(u[0], FileInterval))
+
+    # they're still the same though
+    assert(i == f == u)
+
+    # just for coverage
+    assert_raises(IntervalError, fiset("[]")[0].subset, f[0].start, f[0].end)

nilmdb/test_interval.py

@@ -0,0 +1,189 @@
+from nilmdb import Interval, IntervalSet, IntervalError
+from datetime import datetime
+from nose.tools import assert_raises
+import itertools 
+
+def test_interval():
+    """Test the Interval class"""
+    d1 = datetime.strptime("19801205","%Y%m%d")
+    d2 = datetime.strptime("19900216","%Y%m%d")
+    d3 = datetime.strptime("20111205","%Y%m%d")
+
+    # basic construction
+    i = Interval(d1, d1)
+    i = Interval(d1, d3)
+    assert(i.start == d1)
+    assert(i.end == d3)
+
+    # assignment should work
+    i.start = d2
+    try:
+        i.end = d1
+        raise Exception("should have died there")
+    except IntervalError:
+        pass
+    i.start = d1
+    i.end = d2
+
+    # end before start
+    assert_raises(IntervalError, Interval, d3, d1)
+
+    # wrong type
+    assert_raises(IntervalError, Interval, 1, 2)
+
+    # compare
+    assert(Interval(d1, d2) == Interval(d1, d2))
+    assert(Interval(d1, d2) < Interval(d1, d3))
+    assert(Interval(d1, d3) > Interval(d1, d2))
+    assert(Interval(d1, d2) < Interval(d2, d3))
+    assert(Interval(d1, d3) < Interval(d2, d3))
+    assert(Interval(d2, d2) > Interval(d1, d3))
+    assert(Interval(d3, d3) == Interval(d3, d3))
+    assert_raises(TypeError, cmp, i, 123)
+
+    # subset
+    assert(Interval(d1, d3).subset(d1, d2) == Interval(d1, d2))
+    assert_raises(IntervalError, Interval(d2, d3).subset, d1, d2)
+
+    # append
+    assert(Interval(d1, d2).is_adjacent(Interval(d2,d3)))
+    assert(Interval(d2, d3).is_adjacent(Interval(d1,d2)))
+    assert(not Interval(d2, d3).is_adjacent(Interval(d1,d3)))
+    assert_raises(TypeError, Interval(d1, d2).is_adjacent, 1)
+
+    # misc
+    assert(repr(i) == repr(eval(repr(i).replace("datetime.",""))))
+    assert(str(i) == "[1980-12-05 00:00:00 -> 1990-02-16 00:00:00]")
+
+def test_interval_intersect():
+    """Test Interval intersections"""
+    dates = [ datetime.strptime(year, "%y") for year in [ "00", "01", "02", "03" ] ]
+    perm = list(itertools.permutations(dates, 2))
+    prod = list(itertools.product(perm, perm))
+    should_intersect = {
+        False: [4, 5, 8, 20, 48, 56, 60, 96, 97, 100],
+        True: [0, 1, 2, 12, 13, 14, 16, 17, 24, 25, 26, 28, 29,
+               32, 49, 50, 52, 53, 61, 62, 64, 65, 68, 98, 101, 104]}
+    for i,((a,b),(c,d)) in enumerate(prod):
+        try:
+            i1 = Interval(a, b)
+            i2 = Interval(c, d)
+            assert(i1.intersects(i2) == i2.intersects(i1))
+            assert(i in should_intersect[i1.intersects(i2)])
+        except IntervalError:
+            assert(i not in should_intersect[True] and
+                   i not in should_intersect[False])
+    assert_raises(TypeError, i1.intersects, 1234)
+
+def test_intervalset_construct():
+    """Test interval set construction"""
+    dates = [ datetime.strptime(year, "%y") for year in [ "00", "01", "02", "03" ]]
+
+    a = Interval(dates[0], dates[1])
+    b = Interval(dates[1], dates[2])
+    c = Interval(dates[0], dates[2])
+    d = Interval(dates[2], dates[3])
+
+    iseta = IntervalSet(a)
+    isetb = IntervalSet([a, b])
+    isetc = IntervalSet([a])
+    assert(iseta != isetb)
+    assert(iseta == isetc)
+    assert(iseta != 3)
+    assert(IntervalSet(a) != IntervalSet(b))
+
+    print iseta == None
+    assert_raises(TypeError, cmp, iseta, isetb)
+    assert_raises(IntervalError, IntervalSet, [a, b, c])
+    assert_raises(TypeError, IntervalSet, [1, 2])
+
+    iset = IntervalSet(isetb)   # test iterator
+    assert(iset == isetb)
+    assert(len(iset) == 2)
+    assert(len(IntervalSet()) == 0)
+
+    # Test adding
+    iset = IntervalSet(a)
+    iset += IntervalSet(b)
+    assert(iset == IntervalSet([a, b]))
+    iset = IntervalSet(a)
+    iset += b
+    assert(iset == IntervalSet([a, b]))
+    iset = IntervalSet(a) + IntervalSet(b)
+    assert(iset == IntervalSet([a, b]))
+    iset = IntervalSet(b) + a
+    assert(iset == IntervalSet([a, b]))
+
+    # A set consisting of [0-1],[1-2] should match a set consisting of [0-2]
+    assert(IntervalSet([a,b]) == IntervalSet([c]))
+    # Etc
+    assert(IntervalSet([a,d]) != IntervalSet([c]))
+    assert(IntervalSet([c]) != IntervalSet([a,d]))
+    assert(IntervalSet([c,d]) != IntervalSet([b,d]))
+    
+    # misc
+    assert(repr(iset) == repr(eval(repr(iset).replace("datetime.",""))))
+
+def iset(string):
+    """Build an IntervalSet from a string, for testing purposes
+
+    Each character is a year
+    [ = interval start
+    | = interval end + adjacent start
+    ] = interval end
+    anything else is ignored
+    """
+    iset = IntervalSet()
+    for i, c in enumerate(string):
+        day = datetime.strptime("{0:04d}".format(i+2000), "%Y")
+        if (c == "["):
+            start = day
+        elif (c == "|"):
+            iset += Interval(start, day)
+            start = day
+        elif (c == "]"):
+            iset += Interval(start, day)
+            del start
+    return iset
+
+def test_intervalset_iset():
+    """Test basic iset construction"""
+    assert(iset("  [----]   ") ==
+           iset("  [-|--]   "))
+
+    assert(iset("[]  [--]   ") +
+           iset(" []    [--]") ==
+           iset("[|] [-----]"))
+
+def test_intervalset_intsersect():
+    """Test intersection (&)"""
+    assert_raises(TypeError, iset("[--]").__and__, 1234)
+    
+    assert(iset("[---------]") &
+           iset(" [---]     ") ==
+           iset(" [---]     "))
+
+    assert(iset(" [---]     ") &
+           iset("[---------]") ==
+           iset(" [---]     "))
+
+    assert(iset("    [-----]") &
+           iset(" [-----]   ") ==
+           iset("    [--]   "))
+
+    assert(iset("      [---]") &
+           iset(" [--]      ") ==
+           iset("           "))
+
+    assert(iset("    [-|---]") &
+           iset(" [-----|-] ") ==
+           iset("    [----] "))
+
+    assert(iset("    [-|-]  ") &
+           iset(" [-|--|--] ") ==
+           iset("    [---]  "))
+
+    assert(iset(" [----][--]") &
+           iset("[-] [--] []") ==
+           iset(" [] [-]  []"))
+

nilmdb/utils/__init__.py

@@ -1,10 +0,0 @@
-"""NilmDB utilities"""
-
-from nilmdb.utils.timer import Timer
-from nilmdb.utils.iteratorizer import Iteratorizer
-from nilmdb.utils.serializer import Serializer
-from nilmdb.utils.lrucache import lru_cache
-from nilmdb.utils.diskusage import du, human_size
-from nilmdb.utils.mustclose import must_close
-from nilmdb.utils.urllib import urlencode
-from nilmdb.utils import atomic

nilmdb/utils/atomic.py

@@ -1,26 +0,0 @@
-# Atomic file writing helper.
-
-import os
-
-def replace_file(filename, content):
-    """Attempt to atomically and durably replace the filename with the
-    given contents.  This is intended to be 'pretty good on most
-    OSes', but not necessarily bulletproof."""
-
-    newfilename = filename + ".new"
-
-    # Write to new file, flush it
-    with open(newfilename, "wb") as f:
-        f.write(content)
-        f.flush()
-        os.fsync(f.fileno())
-
-    # Move new file over old one
-    try:
-        os.rename(newfilename, filename)
-    except OSError: # pragma: no cover
-        # Some OSes might not support renaming over an existing file.
-        # This is definitely NOT atomic!
-        os.remove(filename)
-        os.rename(newfilename, filename)
-

nilmdb/utils/datetime_tz/__init__.py

@@ -1,710 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2009 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# Disable the invalid name warning as we are inheriting from a standard library
-# object.
-# pylint: disable-msg=C6409,W0212
-
-"""A version of the datetime module which *cares* about timezones.
-
-This module will never return a naive datetime object. This requires the module
-know your local timezone, which it tries really hard to figure out.
-
-You can override the detection by using the datetime.tzaware.defaulttz_set
-method. It the module is unable to figure out the timezone itself this method
-*must* be called before the normal module is imported. If done before importing
-it can also speed up the time taken to import as the defaulttz will no longer
-try and do the detection.
-"""
-
-__author__ = "tansell@google.com (Tim Ansell)"
-
-import calendar
-import datetime
-import os
-import os.path
-import re
-import time
-import warnings
-import dateutil.parser
-import dateutil.relativedelta
-import dateutil.tz
-import pytz
-import pytz_abbr
-
-
-try:
-  # pylint: disable-msg=C6204
-  import functools
-except ImportError, e:
-
-  class functools(object):
-    """Fake replacement for a full functools."""
-
-    # pylint: disable-msg=W0613
-    @staticmethod
-    def wraps(f, *args, **kw):
-      return f
-
-
-# Need to patch pytz.utc to have a _utcoffset so you can normalize/localize
-# using it.
-pytz.utc._utcoffset = datetime.timedelta()
-
-
-timedelta = datetime.timedelta
-
-
-def _tzinfome(tzinfo):
-  """Gets a tzinfo object from a string.
-
-  Args:
-    tzinfo: A string (or string like) object, or a datetime.tzinfo object.
-
-  Returns:
-    An datetime.tzinfo object.
-
-  Raises:
-    UnknownTimeZoneError: If the timezone given can't be decoded.
-  """
-  if not isinstance(tzinfo, datetime.tzinfo):
-    try:
-      tzinfo = pytz.timezone(tzinfo)
-    except AttributeError:
-      raise pytz.UnknownTimeZoneError("Unknown timezone! %s" % tzinfo)
-  return tzinfo
-
-
-# Our "local" timezone
-_localtz = None
-
-
-def localtz():
-  """Get the local timezone.
-
-  Returns:
-    The localtime timezone as a tzinfo object.
-  """
-  # pylint: disable-msg=W0603
-  global _localtz
-  if _localtz is None:
-    _localtz = detect_timezone()
-  return _localtz
-
-
-def localtz_set(timezone):
-  """Set the local timezone."""
-  # pylint: disable-msg=W0603
-  global _localtz
-  _localtz = _tzinfome(timezone)
-
-
-def detect_timezone():
-  """Try and detect the timezone that Python is currently running in.
-
-  We have a bunch of different methods for trying to figure this out (listed in
-  order they are attempted).
-    * Try TZ environment variable.
-    * Try and find /etc/timezone file (with timezone name).
-    * Try and find /etc/localtime file (with timezone data).
-    * Try and match a TZ to the current dst/offset/shortname.
-
-  Returns:
-    The detected local timezone as a tzinfo object
-
-  Raises:
-    pytz.UnknownTimeZoneError: If it was unable to detect a timezone.
-  """
-  # First we try the TZ variable
-  tz = _detect_timezone_environ()
-  if tz is not None:
-    return tz
-
-  # Second we try /etc/timezone and use the value in that
-  tz = _detect_timezone_etc_timezone()
-  if tz is not None:
-    return tz
-
-  # Next we try and see if something matches the tzinfo in /etc/localtime
-  tz = _detect_timezone_etc_localtime()
-  if tz is not None:
-    return tz
-
-  # Next we try and use a similiar method to what PHP does.
-  # We first try to search on time.tzname, time.timezone, time.daylight to
-  # match a pytz zone.
-  warnings.warn("Had to fall back to worst detection method (the 'PHP' "
-                "method).")
-
-  tz = _detect_timezone_php()
-  if tz is not None:
-    return tz
-
-  raise pytz.UnknownTimeZoneError("Unable to detect your timezone!")
-
-
-def _detect_timezone_environ():
-  if "TZ" in os.environ:
-    try:
-      return pytz.timezone(os.environ["TZ"])
-    except (IOError, pytz.UnknownTimeZoneError):
-      warnings.warn("You provided a TZ environment value (%r) we did not "
-                    "understand!" % os.environ["TZ"])
-
-
-def _detect_timezone_etc_timezone():
-  if os.path.exists("/etc/timezone"):
-    try:
-      tz = file("/etc/timezone").read().strip()
-      try:
-        return pytz.timezone(tz)
-      except (IOError, pytz.UnknownTimeZoneError), ei:
-        warnings.warn("Your /etc/timezone file references a timezone (%r) that"
-                      " is not valid (%r)." % (tz, ei))
-
-    # Problem reading the /etc/timezone file
-    except IOError, eo:
-      warnings.warn("Could not access your /etc/timezone file: %s" % eo)
-
-
-def _detect_timezone_etc_localtime():
-  matches = []
-  if os.path.exists("/etc/localtime"):
-    localtime = pytz.tzfile.build_tzinfo("/etc/localtime",
-                                         file("/etc/localtime"))
-
-    # See if we can find a "Human Name" for this..
-    for tzname in pytz.all_timezones:
-      tz = _tzinfome(tzname)
-
-      if dir(tz) != dir(localtime):
-        continue
-
-      for attrib in dir(tz):
-        # Ignore functions and specials
-        if callable(getattr(tz, attrib)) or attrib.startswith("__"):
-          continue
-
-        # This will always be different
-        if attrib == "zone" or attrib == "_tzinfos":
-          continue
-
-        if getattr(tz, attrib) != getattr(localtime, attrib):
-          break
-
-      # We get here iff break didn't happen, i.e. no meaningful attributes
-      # differ between tz and localtime
-      else:
-        matches.append(tzname)
-
-    if len(matches) == 1:
-      return _tzinfome(matches[0])
-    else:
-      # Warn the person about this!
-      warning = "Could not get a human name for your timezone: "
-      if len(matches) > 1:
-        warning += ("We detected multiple matches for your /etc/localtime. "
-                    "(Matches where %s)" % matches)
-        return _tzinfome(matches[0])
-      else:
-        warning += "We detected no matches for your /etc/localtime."
-      warnings.warn(warning)
-
-      # Register /etc/localtime as the timezone loaded.
-      pytz._tzinfo_cache['/etc/localtime'] = localtime
-      return localtime
-
-
-def _detect_timezone_php():
-  tomatch = (time.tzname[0], time.timezone, time.daylight)
-  now = datetime.datetime.now()
-
-  matches = []
-  for tzname in pytz.all_timezones:
-    try:
-      tz = pytz.timezone(tzname)
-    except IOError:
-      continue
-
-    try:
-      indst = tz.localize(now).timetuple()[-1]
-
-      if tomatch == (tz._tzname, -tz._utcoffset.seconds, indst):
-        matches.append(tzname)
-
-    # pylint: disable-msg=W0704
-    except AttributeError:
-      pass
-
-  if len(matches) > 1:
-    warnings.warn("We detected multiple matches for the timezone, choosing "
-                  "the first %s. (Matches where %s)" % (matches[0], matches))
-    return pytz.timezone(matches[0])
-
-
-class datetime_tz(datetime.datetime):
-  """An extension of the inbuilt datetime adding more functionality.
-
-  The extra functionality includes:
-    * Partial parsing support (IE 2006/02/30 matches %Y/%M/%D %H:%M)
-    * Full integration with pytz (just give it the string of the timezone!)
-    * Proper support for going to/from Unix timestamps (which are in UTC!).
-  """
-  __slots__ = ["is_dst"]
-
-  def __new__(cls, *args, **kw):
-    args = list(args)
-    if not args:
-      raise TypeError("Not enough arguments given.")
-
-    # See if we are given a tzinfo object...
-    tzinfo = None
-    if isinstance(args[-1], (datetime.tzinfo, basestring)):
-      tzinfo = _tzinfome(args.pop(-1))
-    elif kw.get("tzinfo", None) is not None:
-      tzinfo = _tzinfome(kw.pop("tzinfo"))
-
-    # Create a datetime object if we don't have one
-    if isinstance(args[0], datetime.datetime):
-      # Convert the datetime instance to a datetime object.
-      newargs = (list(args[0].timetuple()[0:6]) +
-                 [args[0].microsecond, args[0].tzinfo])
-      dt = datetime.datetime(*newargs)
-
-      if tzinfo is None and dt.tzinfo is None:
-        raise TypeError("Must specify a timezone!")
-
-      if tzinfo is not None and dt.tzinfo is not None:
-        raise TypeError("Can not give a timezone with timezone aware"
-                        " datetime object! (Use localize.)")
-    else:
-      dt = datetime.datetime(*args, **kw)
-
-    if dt.tzinfo is not None:
-      # Re-normalize the dt object
-      dt = dt.tzinfo.normalize(dt)
-
-    else:
-      if tzinfo is None:
-        tzinfo = localtz()
-
-      try:
-        dt = tzinfo.localize(dt, is_dst=None)
-      except pytz.AmbiguousTimeError:
-        is_dst = None
-        if "is_dst" in kw:
-          is_dst = kw.pop("is_dst")
-
-        try:
-          dt = tzinfo.localize(dt, is_dst)
-        except IndexError:
-          raise pytz.AmbiguousTimeError("No such time exists!")
-
-    newargs = list(dt.timetuple()[0:6])+[dt.microsecond, dt.tzinfo]
-    obj = datetime.datetime.__new__(cls, *newargs)
-    obj.is_dst = obj.dst() != datetime.timedelta(0)
-    return obj
-
-  def asdatetime(self, naive=True):
-    """Return this datetime_tz as a datetime object.
-
-    Args:
-      naive: Return *without* any tz info.
-
-    Returns:
-      This datetime_tz as a datetime object.
-    """
-    args = list(self.timetuple()[0:6])+[self.microsecond]
-    if not naive:
-      args.append(self.tzinfo)
-    return datetime.datetime(*args)
-
-  def asdate(self):
-    """Return this datetime_tz as a date object.
-
-    Returns:
-      This datetime_tz as a date object.
-    """
-    return datetime.date(self.year, self.month, self.day)
-
-  def totimestamp(self):
-    """Convert this datetime object back to a unix timestamp.
-
-    The Unix epoch is the time 00:00:00 UTC on January 1, 1970.
-
-    Returns:
-      Unix timestamp.
-    """
-    return calendar.timegm(self.utctimetuple())+1e-6*self.microsecond
-
-  def astimezone(self, tzinfo):
-    """Returns a version of this timestamp converted to the given timezone.
-
-    Args:
-      tzinfo: Either a datetime.tzinfo object or a string (which will be looked
-              up in pytz.
-
-    Returns:
-      A datetime_tz object in the given timezone.
-    """
-    # Assert we are not a naive datetime object
-    assert self.tzinfo is not None
-
-    tzinfo = _tzinfome(tzinfo)
-
-    d = self.asdatetime(naive=False).astimezone(tzinfo)
-    return datetime_tz(d)
-
-  # pylint: disable-msg=C6113
-  def replace(self, **kw):
-    """Return datetime with new specified fields given as arguments.
-
-    For example, dt.replace(days=4) would return a new datetime_tz object with
-    exactly the same as dt but with the days attribute equal to 4.
-
-    Any attribute can be replaced, but tzinfo can not be set to None.
-
-    Args:
-      Any datetime_tz attribute.
-
-    Returns:
-      A datetime_tz object with the attributes replaced.
-
-    Raises:
-      TypeError: If the given replacement is invalid.
-    """
-    if "tzinfo" in kw:
-      if kw["tzinfo"] is None:
-        raise TypeError("Can not remove the timezone use asdatetime()")
-
-    is_dst = None
-    if "is_dst" in kw:
-      is_dst = kw["is_dst"]
-      del kw["is_dst"]
-    else:
-      # Use our own DST setting..
-      is_dst = self.is_dst
-
-    replaced = self.asdatetime().replace(**kw)
-
-    return datetime_tz(replaced, tzinfo=self.tzinfo.zone, is_dst=is_dst)
-
-  # pylint: disable-msg=C6310
-  @classmethod
-  def smartparse(cls, toparse, tzinfo=None):
-    """Method which uses dateutil.parse and extras to try and parse the string.
-
-    Valid dates are found at:
-     http://labix.org/python-dateutil#head-1443e0f14ad5dff07efd465e080d1110920673d8-2
-
-    Other valid formats include:
-      "now" or "today"
-      "yesterday"
-      "tommorrow"
-      "5 minutes ago"
-      "10 hours ago"
-      "10h5m ago"
-      "start of yesterday"
-      "end of tommorrow"
-      "end of 3rd of March"
-
-    Args:
-      toparse: The string to parse.
-      tzinfo: Timezone for the resultant datetime_tz object should be in.
-              (Defaults to your local timezone.)
-
-    Returns:
-      New datetime_tz object.
-
-    Raises:
-      ValueError: If unable to make sense of the input.
-    """
-    # Default for empty fields are:
-    #  year/month/day == now
-    #  hour/minute/second/microsecond == 0
-    toparse = toparse.strip()
-
-    if tzinfo is None:
-      dt = cls.now()
-    else:
-      dt = cls.now(tzinfo)
-
-    default = dt.replace(hour=0, minute=0, second=0, microsecond=0)
-
-    # Remove "start of " and "end of " prefix in the string
-    if toparse.lower().startswith("end of "):
-      toparse = toparse[7:].strip()
-
-      dt += datetime.timedelta(days=1)
-      dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
-      dt -= datetime.timedelta(microseconds=1)
-
-      default = dt
-
-    elif toparse.lower().startswith("start of "):
-      toparse = toparse[9:].strip()
-
-      dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
-      default = dt
-
-    # Handle strings with "now", "today", "yesterday", "tomorrow" and "ago".
-    # Need to use lowercase
-    toparselower = toparse.lower()
-
-    if toparselower in ["now", "today"]:
-      pass
-
-    elif toparselower == "yesterday":
-      dt -= datetime.timedelta(days=1)
-
-    elif toparselower == "tommorrow":
-      dt += datetime.timedelta(days=1)
-
-    elif "ago" in toparselower:
-      # Remove the "ago" bit
-      toparselower = toparselower[:-3]
-      # Replace all "a day and an hour" with "1 day 1 hour"
-      toparselower = toparselower.replace("a ", "1 ")
-      toparselower = toparselower.replace("an ", "1 ")
-      toparselower = toparselower.replace(" and ", " ")
-
-      # Match the following
-      # 1 hour ago
-      # 1h ago
-      # 1 h ago
-      # 1 hour ago
-      # 2 hours ago
-      # Same with minutes, seconds, etc.
-
-      tocheck = ("seconds", "minutes", "hours", "days", "weeks", "months",
-                 "years")
-      result = {}
-      for match in re.finditer("([0-9]+)([^0-9]*)", toparselower):
-        amount = int(match.group(1))
-        unit = match.group(2).strip()
-
-        for bit in tocheck:
-          regex = "^([%s]|((%s)s?))$" % (
-              bit[0], bit[:-1])
-
-          bitmatch = re.search(regex, unit)
-          if bitmatch:
-            result[bit] = amount
-            break
-        else:
-          raise ValueError("Was not able to parse date unit %r!" % unit)
-
-      delta = dateutil.relativedelta.relativedelta(**result)
-      dt -= delta
-
-    else:
-      # Handle strings with normal datetime format, use original case.
-      dt = dateutil.parser.parse(toparse, default=default.asdatetime(),
-                                 tzinfos=pytz_abbr.tzinfos)
-      if dt is None:
-        raise ValueError("Was not able to parse date!")
-
-      if dt.tzinfo is pytz_abbr.unknown:
-        dt = dt.replace(tzinfo=None)
-
-      if dt.tzinfo is None:
-        if tzinfo is None:
-          tzinfo = localtz()
-        dt = cls(dt, tzinfo)
-      else:
-        if isinstance(dt.tzinfo, pytz_abbr.tzabbr):
-          abbr = dt.tzinfo
-          dt = dt.replace(tzinfo=None)
-          dt = cls(dt, abbr.zone, is_dst=abbr.dst)
-
-        dt = cls(dt)
-
-    return dt
-
-  @classmethod
-  def utcfromtimestamp(cls, timestamp):
-    """Returns a datetime object of a given timestamp (in UTC)."""
-    obj = datetime.datetime.utcfromtimestamp(timestamp)
-    obj = pytz.utc.localize(obj)
-    return cls(obj)
-
-  @classmethod
-  def fromtimestamp(cls, timestamp):
-    """Returns a datetime object of a given timestamp (in local tz)."""
-    d = cls.utcfromtimestamp(timestamp)
-    return d.astimezone(localtz())
-
-  @classmethod
-  def utcnow(cls):
-    """Return a new datetime representing UTC day and time."""
-    obj = datetime.datetime.utcnow()
-    obj = cls(obj, tzinfo=pytz.utc)
-    return obj
-
-  @classmethod
-  def now(cls, tzinfo=None):
-    """[tz] -> new datetime with tz's local day and time."""
-    obj = cls.utcnow()
-    if tzinfo is None:
-      tzinfo = localtz()
-    return obj.astimezone(tzinfo)
-
-  today = now
-
-  @staticmethod
-  def fromordinal(ordinal):
-    raise SyntaxError("Not enough information to create a datetime_tz object "
-                      "from an ordinal. Please use datetime.date.fromordinal")
-
-
-class iterate(object):
-  """Helpful iterators for working with datetime_tz objects."""
-
-  @staticmethod
-  def between(start, delta, end=None):
-    """Return an iterator between this date till given end point.
-
-    Example usage:
-      >>> d = datetime_tz.smartparse("5 days ago")
-      2008/05/12 11:45
-      >>> for i in d.between(timedelta(days=1), datetime_tz.now()):
-      >>>    print i
-      2008/05/12 11:45
-      2008/05/13 11:45
-      2008/05/14 11:45
-      2008/05/15 11:45
-      2008/05/16 11:45
-
-    Args:
-      start: The date to start at.
-      delta: The interval to iterate with.
-      end: (Optional) Date to end at. If not given the iterator will never
-           terminate.
-
-    Yields:
-      datetime_tz objects.
-    """
-    toyield = start
-    while end is None or toyield < end:
-      yield toyield
-      toyield += delta
-
-  @staticmethod
-  def weeks(start, end=None):
-    """Iterate over the weeks between the given datetime_tzs.
-
-    Args:
-      start: datetime_tz to start from.
-      end: (Optional) Date to end at, if not given the iterator will never
-           terminate.
-
-    Returns:
-      An iterator which generates datetime_tz objects a week apart.
-    """
-    return iterate.between(start, datetime.timedelta(days=7), end)
-
-  @staticmethod
-  def days(start, end=None):
-    """Iterate over the days between the given datetime_tzs.
-
-    Args:
-      start: datetime_tz to start from.
-      end: (Optional) Date to end at, if not given the iterator will never
-           terminate.
-
-    Returns:
-      An iterator which generates datetime_tz objects a day apart.
-    """
-    return iterate.between(start, datetime.timedelta(days=1), end)
-
-  @staticmethod
-  def hours(start, end=None):
-    """Iterate over the hours between the given datetime_tzs.
-
-    Args:
-      start: datetime_tz to start from.
-      end: (Optional) Date to end at, if not given the iterator will never
-           terminate.
-
-    Returns:
-      An iterator which generates datetime_tz objects a hour apart.
-    """
-    return iterate.between(start, datetime.timedelta(hours=1), end)
-
-  @staticmethod
-  def minutes(start, end=None):
-    """Iterate over the minutes between the given datetime_tzs.
-
-    Args:
-      start: datetime_tz to start from.
-      end: (Optional) Date to end at, if not given the iterator will never
-           terminate.
-
-    Returns:
-      An iterator which generates datetime_tz objects a minute apart.
-    """
-    return iterate.between(start, datetime.timedelta(minutes=1), end)
-
-  @staticmethod
-  def seconds(start, end=None):
-    """Iterate over the seconds between the given datetime_tzs.
-
-    Args:
-      start: datetime_tz to start from.
-      end: (Optional) Date to end at, if not given the iterator will never
-           terminate.
-
-    Returns:
-      An iterator which generates datetime_tz objects a second apart.
-    """
-    return iterate.between(start, datetime.timedelta(minutes=1), end)
-
-
-def _wrap_method(name):
-  """Wrap a method.
-
-  Patch a method which might return a datetime.datetime to return a
-  datetime_tz.datetime_tz instead.
-
-  Args:
-    name: The name of the method to patch
-  """
-  method = getattr(datetime.datetime, name)
-
-  # Have to give the second argument as method has no __module__ option.
-  @functools.wraps(method, ("__name__", "__doc__"), ())
-  def wrapper(*args, **kw):
-    r = method(*args, **kw)
-
-    if isinstance(r, datetime.datetime) and not isinstance(r, datetime_tz):
-      r = datetime_tz(r)
-    return r
-
-  setattr(datetime_tz, name, wrapper)
-
-for methodname in ["__add__", "__radd__", "__rsub__", "__sub__", "combine"]:
-
-  # Make sure we have not already got an override for this method
-  assert methodname not in datetime_tz.__dict__
-
-  _wrap_method(methodname)
-
-
-__all__ = ['datetime_tz', 'detect_timezone', 'iterate', 'localtz',
-    'localtz_set', 'timedelta', '_detect_timezone_environ',
-    '_detect_timezone_etc_localtime', '_detect_timezone_etc_timezone',
-    '_detect_timezone_php']

nilmdb/utils/datetime_tz/pytz_abbr.py

@@ -1,230 +0,0 @@
-#!/usr/bin/python2.4
-# -*- coding: utf-8 -*-
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
-#
-
-"""
-Common time zone acronyms/abbreviations for use with the datetime_tz module.
-
-*WARNING*: There are lots of caveats when using this module which are listed
-below.
-
-CAVEAT 1: The acronyms/abbreviations are not globally unique, they are not even
-unique within a region. For example, EST can mean any of,
-  Eastern Standard Time in Australia (which is 10 hour ahead of UTC)
-  Eastern Standard Time in North America (which is 5 hours behind UTC)
-
-Where there are two abbreviations the more popular one will appear in the all
-dictionary, while the less common one will only appear in that countries region
-dictionary. IE If using all, EST will be mapped to Eastern Standard Time in
-North America.
-
-CAVEAT 2: Many of the acronyms don't map to a neat Oslon timezones. For example,
-Eastern European Summer Time (EEDT) is used by many different countries in
-Europe *at different times*! If the acronym does not map neatly to one zone it
-is mapped to the Etc/GMT+-XX Oslon zone. This means that any date manipulations
-can end up with idiot things like summer time in the middle of winter.
-
-CAVEAT 3: The Summer/Standard time difference is really important! For an hour
-each year it is needed to determine which time you are actually talking about.
-    2002-10-27 01:20:00 EST != 2002-10-27 01:20:00 EDT
-"""
-
-import datetime
-import pytz
-import pytz.tzfile
-
-
-class tzabbr(datetime.tzinfo):
-  """A timezone abbreviation.
-
-  *WARNING*: This is not a tzinfo implementation! Trying to use this as tzinfo
-  object will result in failure.  We inherit from datetime.tzinfo so we can get
-  through the dateutil checks.
-  """
-  pass
-
-
-# A "marker" tzinfo object which is used to signify an unknown timezone.
-unknown = datetime.tzinfo(0)
-
-
-regions = {'all': {}, 'military': {}}
-# Create a special alias for the all and military regions
-all = regions['all']
-military = regions['military']
-
-
-def tzabbr_register(abbr, name, region, zone, dst):
-  """Register a new timezone abbreviation in the global registry.
-
-  If another abbreviation with the same name has already been registered it new
-  abbreviation will only be registered in region specific dictionary.
-  """
-  newabbr = tzabbr()
-  newabbr.abbr = abbr
-  newabbr.name = name
-  newabbr.region = region
-  newabbr.zone = zone
-  newabbr.dst = dst
-
-  if abbr not in all:
-    all[abbr] = newabbr
-
-  if not region in regions:
-    regions[region] = {}
-
-  assert abbr not in regions[region]
-  regions[region][abbr] = newabbr
-
-
-def tzinfos_create(use_region):
-  abbrs = regions[use_region]
-
-  def tzinfos(abbr, offset):
-    if abbr:
-      if abbr in abbrs:
-        result = abbrs[abbr]
-        if offset:
-          # FIXME: Check the offset matches the abbreviation we just selected.
-          pass
-        return result
-      else:
-        raise ValueError, "Unknown timezone found %s" % abbr
-    if offset == 0:
-      return pytz.utc
-    if offset:
-      return pytz.FixedOffset(offset/60)
-    return unknown
-
-  return tzinfos
-
-
-# Create a special alias for the all tzinfos
-tzinfos = tzinfos_create('all')
-
-
-# Create the abbreviations.
-# *WARNING*: Order matters!
-tzabbr_register("A", u"Alpha Time Zone", u"Military", "Etc/GMT-1", False)
-tzabbr_register("ACDT", u"Australian Central Daylight Time", u"Australia",
-                "Australia/Adelaide", True)
-tzabbr_register("ACST", u"Australian Central Standard Time", u"Australia",
-                "Australia/Adelaide", False)
-tzabbr_register("ADT", u"Atlantic Daylight Time", u"North America",
-                "America/Halifax", True)
-tzabbr_register("AEDT", u"Australian Eastern Daylight Time", u"Australia",
-                "Australia/Sydney", True)
-tzabbr_register("AEST", u"Australian Eastern Standard Time", u"Australia",
-                "Australia/Sydney", False)
-tzabbr_register("AKDT", u"Alaska Daylight Time", u"North America",
-                "US/Alaska", True)
-tzabbr_register("AKST", u"Alaska Standard Time", u"North America",
-                "US/Alaska", False)
-tzabbr_register("AST", u"Atlantic Standard Time", u"North America",
-                "America/Halifax", False)
-tzabbr_register("AWDT", u"Australian Western Daylight Time", u"Australia",
-                "Australia/West", True)
-tzabbr_register("AWST", u"Australian Western Standard Time", u"Australia",
-                "Australia/West", False)
-tzabbr_register("B", u"Bravo Time Zone", u"Military", "Etc/GMT-2", False)
-tzabbr_register("BST", u"British Summer Time", u"Europe", "Europe/London", True)
-tzabbr_register("C", u"Charlie Time Zone", u"Military", "Etc/GMT-2", False)
-tzabbr_register("CDT", u"Central Daylight Time", u"North America",
-                "US/Central", True)
-tzabbr_register("CEDT", u"Central European Daylight Time", u"Europe",
-                "Etc/GMT+2", True)
-tzabbr_register("CEST", u"Central European Summer Time", u"Europe",
-                "Etc/GMT+2", True)
-tzabbr_register("CET", u"Central European Time", u"Europe", "Etc/GMT+1", False)
-tzabbr_register("CST", u"Central Standard Time", u"North America",
-                "US/Central", False)
-tzabbr_register("CXT", u"Christmas Island Time", u"Australia",
-                "Indian/Christmas", False)
-tzabbr_register("D", u"Delta Time Zone", u"Military", "Etc/GMT-2", False)
-tzabbr_register("E", u"Echo Time Zone", u"Military", "Etc/GMT-2", False)
-tzabbr_register("EDT", u"Eastern Daylight Time", u"North America",
-                "US/Eastern", True)
-tzabbr_register("EEDT", u"Eastern European Daylight Time", u"Europe",
-                "Etc/GMT+3", True)
-tzabbr_register("EEST", u"Eastern European Summer Time", u"Europe",
-                "Etc/GMT+3", True)
-tzabbr_register("EET", u"Eastern European Time", u"Europe", "Etc/GMT+2", False)
-tzabbr_register("EST", u"Eastern Standard Time", u"North America",
-                "US/Eastern", False)
-tzabbr_register("F", u"Foxtrot Time Zone", u"Military", "Etc/GMT-6", False)
-tzabbr_register("G", u"Golf Time Zone", u"Military", "Etc/GMT-7", False)
-tzabbr_register("GMT", u"Greenwich Mean Time", u"Europe", pytz.utc, False)
-tzabbr_register("H", u"Hotel Time Zone", u"Military", "Etc/GMT-8", False)
-#tzabbr_register("HAA", u"Heure Avancée de l'Atlantique", u"North America", u"UTC - 3 hours")
-#tzabbr_register("HAC", u"Heure Avancée du Centre", u"North America", u"UTC - 5 hours")
-tzabbr_register("HADT", u"Hawaii-Aleutian Daylight Time", u"North America",
-                "Pacific/Honolulu", True)
-#tzabbr_register("HAE", u"Heure Avancée de l'Est", u"North America", u"UTC - 4 hours")
-#tzabbr_register("HAP", u"Heure Avancée du Pacifique", u"North America", u"UTC - 7 hours")
-#tzabbr_register("HAR", u"Heure Avancée des Rocheuses", u"North America", u"UTC - 6 hours")
-tzabbr_register("HAST", u"Hawaii-Aleutian Standard Time", u"North America",
-                "Pacific/Honolulu", False)
-#tzabbr_register("HAT", u"Heure Avancée de Terre-Neuve", u"North America", u"UTC - 2:30 hours")
-#tzabbr_register("HAY", u"Heure Avancée du Yukon", u"North America", u"UTC - 8 hours")
-tzabbr_register("HDT", u"Hawaii Daylight Time", u"North America",
-                "Pacific/Honolulu", True)
-#tzabbr_register("HNA", u"Heure Normale de l'Atlantique", u"North America", u"UTC - 4 hours")
-#tzabbr_register("HNC", u"Heure Normale du Centre", u"North America", u"UTC - 6 hours")
-#tzabbr_register("HNE", u"Heure Normale de l'Est", u"North America", u"UTC - 5 hours")
-#tzabbr_register("HNP", u"Heure Normale du Pacifique", u"North America", u"UTC - 8 hours")
-#tzabbr_register("HNR", u"Heure Normale des Rocheuses", u"North America", u"UTC - 7 hours")
-#tzabbr_register("HNT", u"Heure Normale de Terre-Neuve", u"North America", u"UTC - 3:30 hours")
-#tzabbr_register("HNY", u"Heure Normale du Yukon", u"North America", u"UTC - 9 hours")
-tzabbr_register("HST", u"Hawaii Standard Time", u"North America",
-                "Pacific/Honolulu", False)
-tzabbr_register("I", u"India Time Zone", u"Military", "Etc/GMT-9", False)
-tzabbr_register("IST", u"Irish Summer Time", u"Europe", "Europe/Dublin", True)
-tzabbr_register("K", u"Kilo Time Zone", u"Military", "Etc/GMT-10", False)
-tzabbr_register("L", u"Lima Time Zone", u"Military", "Etc/GMT-11", False)
-tzabbr_register("M", u"Mike Time Zone", u"Military", "Etc/GMT-12", False)
-tzabbr_register("MDT", u"Mountain Daylight Time", u"North America",
-                "US/Mountain", True)
-#tzabbr_register("MESZ", u"Mitteleuroäische Sommerzeit", u"Europe", u"UTC + 2 hours")
-#tzabbr_register("MEZ", u"Mitteleuropäische Zeit", u"Europe", u"UTC + 1 hour")
-tzabbr_register("MSD", u"Moscow Daylight Time", u"Europe",
-                "Europe/Moscow", True)
-tzabbr_register("MSK", u"Moscow Standard Time", u"Europe",
-                "Europe/Moscow", False)
-tzabbr_register("MST", u"Mountain Standard Time", u"North America",
-                "US/Mountain", False)
-tzabbr_register("N", u"November Time Zone", u"Military", "Etc/GMT+1", False)
-tzabbr_register("NDT", u"Newfoundland Daylight Time", u"North America",
-                "America/St_Johns", True)
-tzabbr_register("NFT", u"Norfolk (Island) Time", u"Australia",
-                "Pacific/Norfolk", False)
-tzabbr_register("NST", u"Newfoundland Standard Time", u"North America",
-                "America/St_Johns", False)
-tzabbr_register("O", u"Oscar Time Zone", u"Military", "Etc/GMT+2", False)
-tzabbr_register("P", u"Papa Time Zone", u"Military", "Etc/GMT+3", False)
-tzabbr_register("PDT", u"Pacific Daylight Time", u"North America",
-                "US/Pacific", True)
-tzabbr_register("PST", u"Pacific Standard Time", u"North America",
-                "US/Pacific", False)
-tzabbr_register("Q", u"Quebec Time Zone", u"Military", "Etc/GMT+4", False)
-tzabbr_register("R", u"Romeo Time Zone", u"Military", "Etc/GMT+5", False)
-tzabbr_register("S", u"Sierra Time Zone", u"Military", "Etc/GMT+6", False)
-tzabbr_register("T", u"Tango Time Zone", u"Military", "Etc/GMT+7", False)
-tzabbr_register("U", u"Uniform Time Zone", u"Military", "Etc/GMT+8", False)
-tzabbr_register("UTC", u"Coordinated Universal Time", u"Europe",
-                pytz.utc, False)
-tzabbr_register("V", u"Victor Time Zone", u"Military", "Etc/GMT+9", False)
-tzabbr_register("W", u"Whiskey Time Zone", u"Military", "Etc/GMT+10", False)
-tzabbr_register("WDT", u"Western Daylight Time", u"Australia",
-                "Australia/West", True)
-tzabbr_register("WEDT", u"Western European Daylight Time", u"Europe",
-                "Etc/GMT+1", True)
-tzabbr_register("WEST", u"Western European Summer Time", u"Europe",
-                "Etc/GMT+1", True)
-tzabbr_register("WET", u"Western European Time", u"Europe", pytz.utc, False)
-tzabbr_register("WST", u"Western Standard Time", u"Australia",
-                "Australia/West", False)
-tzabbr_register("X", u"X-ray Time Zone", u"Military", "Etc/GMT+11", False)
-tzabbr_register("Y", u"Yankee Time Zone", u"Military", "Etc/GMT+12", False)
-tzabbr_register("Z", u"Zulu Time Zone", u"Military", pytz.utc, False)

nilmdb/utils/diskusage.py

@@ -1,25 +0,0 @@
-import os
-from math import log
-
-def human_size(num):
-    """Human friendly file size"""
-    unit_list = zip(['bytes', 'kiB', 'MiB', 'GiB', 'TiB'], [0, 0, 1, 2, 2])
-    if num > 1:
-        exponent = min(int(log(num, 1024)), len(unit_list) - 1)
-        quotient = float(num) / 1024**exponent
-        unit, num_decimals = unit_list[exponent]
-        format_string = '{:.%sf} {}' % (num_decimals)
-        return format_string.format(quotient, unit)
-    if num == 0: # pragma: no cover
-        return '0 bytes'
-    if num == 1: # pragma: no cover
-        return '1 byte'
-
-def du(path):
-    """Like du -sb, returns total size of path in bytes."""
-    size = os.path.getsize(path)
-    if os.path.isdir(path):
-        for thisfile in os.listdir(path):
-            filepath = os.path.join(path, thisfile)
-            size += du(filepath)
-    return size

nilmdb/utils/iteratorizer.py

@@ -1,99 +0,0 @@
-import Queue
-import threading
-import sys
-import contextlib
-
-# This file provides a context manager that converts a function
-# that takes a callback into a generator that returns an iterable.
-# This is done by running the function in a new thread.
-
-# Based partially on http://stackoverflow.com/questions/9968592/
-
-class IteratorizerThread(threading.Thread):
-    def __init__(self, queue, function, curl_hack):
-        """
-        function: function to execute, which takes the
-        callback (provided by this class) as an argument
-        """
-        threading.Thread.__init__(self)
-        self.function = function
-        self.queue = queue
-        self.die = False
-        self.curl_hack = curl_hack
-
-    def callback(self, data):
-        try:
-            if self.die:
-                raise Exception() # trigger termination
-            self.queue.put((1, data))
-        except:
-            if self.curl_hack:
-                # We can't raise exceptions, because the pycurl
-                # extension module will unconditionally print the
-                # exception itself, and not pass it up to the caller.
-                # Instead, just return a value that tells curl to
-                # abort.  (-1 would be best, in case we were given 0
-                # bytes, but the extension doesn't support that).
-                self.queue.put((2, sys.exc_info()))
-                return 0
-            raise
-
-    def run(self):
-        try:
-            result = self.function(self.callback)
-        except:
-            self.queue.put((2, sys.exc_info()))
-        else:
-            self.queue.put((0, result))
-
-@contextlib.contextmanager
-def Iteratorizer(function, curl_hack = False):
-    """
-    Context manager that takes a function expecting a callback,
-    and provides an iterable that yields the values passed to that
-    callback instead.
-
-    function: function to execute, which takes a callback
-    (provided by this context manager) as an argument
-
-        with iteratorizer(func) as it:
-            for i in it:
-                print 'callback was passed:', i
-        print 'function returned:', it.retval
-    """
-    queue = Queue.Queue(maxsize = 1)
-    thread = IteratorizerThread(queue, function, curl_hack)
-    thread.daemon = True
-    thread.start()
-
-    class iteratorizer_gen(object):
-        def __init__(self, queue):
-            self.queue = queue
-            self.retval = None
-
-        def __iter__(self):
-            return self
-
-        def next(self):
-            (typ, data) = self.queue.get()
-            if typ == 0:
-                # function has returned
-                self.retval = data
-                raise StopIteration
-            elif typ == 1:
-                # data is available
-                return data
-            else:
-                # callback raised an exception
-                raise data[0], data[1], data[2]
-
-    try:
-        yield iteratorizer_gen(queue)
-    finally:
-        # Ask the thread to die, if it's still running.
-        thread.die = True
-        while thread.isAlive():
-            try:
-                queue.get(True, 0.01)
-            except: # pragma: no cover
-                pass

nilmdb/utils/lrucache.py

@@ -1,76 +0,0 @@
-# Memoize a function's return value with a least-recently-used cache
-# Based on:
-#   http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators/
-# with added 'destructor' functionality.
-
-import collections
-import decorator
-
-def lru_cache(size = 10, onremove = None, keys = slice(None)):
-    """Least-recently-used cache decorator.
-
-    @lru_cache(size = 10, onevict = None)
-    def f(...):
-        pass
-
-    Given a function and arguments, memoize its return value.  Up to
-    'size' elements are cached.  'keys' is a slice object that
-    represents which arguments are used as the cache key.
-
-    When evicting a value from the cache, call the function
-    'onremove' with the value that's being evicted.
-
-    Call f.cache_remove(...) to evict the cache entry with the given
-    arguments.  Call f.cache_remove_all() to evict all entries.
-    f.cache_hits and f.cache_misses give statistics.
-    """
-
-    def decorate(func):
-        cache = collections.OrderedDict()	# order: least- to most-recent
-
-        def evict(value):
-            if onremove:
-                onremove(value)
-
-        def wrapper(orig, *args, **kwargs):
-            if kwargs:
-                raise NotImplementedError("kwargs not supported")
-            key = args[keys]
-            try:
-                value = cache.pop(key)
-                orig.cache_hits += 1
-            except KeyError:
-                value = orig(*args)
-                orig.cache_misses += 1
-                if len(cache) >= size:
-                    evict(cache.popitem(0)[1])	# evict LRU cache entry
-            cache[key] = value              	# (re-)insert this key at end
-            return value
-
-        def cache_remove(*args):
-            """Remove the described key from this cache, if present."""
-            key = args
-            if key in cache:
-                evict(cache.pop(key))
-            else:
-                if len(cache) > 0 and len(args) != len(cache.iterkeys().next()):
-                    raise KeyError("trying to remove from LRU cache, but "
-                                   "number of arguments doesn't match the "
-                                   "cache key length")
-
-        def cache_remove_all():
-            for key in cache:
-                evict(cache.pop(key))
-
-        def cache_info():
-            return (func.cache_hits, func.cache_misses)
-
-        new = decorator.decorator(wrapper, func)
-        func.cache_hits = 0
-        func.cache_misses = 0
-        new.cache_info = cache_info
-        new.cache_remove = cache_remove
-        new.cache_remove_all = cache_remove_all
-        return new
-
-    return decorate

nilmdb/utils/mustclose.py

@@ -1,64 +0,0 @@
-from nilmdb.utils.printf import *
-import sys
-import inspect
-import decorator
-
-def must_close(errorfile = sys.stderr, wrap_verify = False):
-    """Class decorator that warns on 'errorfile' at deletion time if
-    the class's close() member wasn't called.
-
-    If 'wrap_verify' is True, every class method is wrapped with a
-    verifier that will raise AssertionError if the .close() method has
-    already been called."""
-    def class_decorator(cls):
-
-        # Helper to replace a class method with a wrapper function,
-        # while maintaining argument specs etc.
-        def wrap_class_method(wrapper_func):
-            method = wrapper_func.__name__
-            if method in cls.__dict__:
-                orig = getattr(cls, method).im_func
-            else:
-                orig = lambda self: None
-            setattr(cls, method, decorator.decorator(wrapper_func, orig))
-
-        @wrap_class_method
-        def __init__(orig, self, *args, **kwargs):
-            ret = orig(self, *args, **kwargs)
-            self.__dict__["_must_close"] = True
-            self.__dict__["_must_close_initialized"] = True
-            return ret
-
-        @wrap_class_method
-        def __del__(orig, self, *args, **kwargs):
-            if "_must_close" in self.__dict__:
-                fprintf(errorfile, "error: %s.close() wasn't called!\n",
-                        self.__class__.__name__)
-            return orig(self, *args, **kwargs)
-
-        @wrap_class_method
-        def close(orig, self, *args, **kwargs):
-            if "_must_close" in self.__dict__:
-                del self._must_close
-            return orig(self, *args, **kwargs)
-
-        # Optionally wrap all other functions
-        def verifier(orig, self, *args, **kwargs):
-            if ("_must_close" not in self.__dict__ and
-                "_must_close_initialized" in self.__dict__):
-                raise AssertionError("called " + str(orig) + " after close")
-            return orig(self, *args, **kwargs)
-        if wrap_verify:
-            for (name, method) in inspect.getmembers(cls, inspect.ismethod):
-                # Skip class methods
-                if method.__self__ is not None:
-                    continue
-                # Skip some methods
-                if name in [ "__del__", "__init__" ]:
-                    continue
-                # Set up wrapper
-                setattr(cls, name, decorator.decorator(verifier,
-                                                       method.im_func))
-
-        return cls
-    return class_decorator

nilmdb/utils/printf.py

@@ -1,9 +0,0 @@
-"""printf, fprintf, sprintf"""
-
-from __future__ import print_function
-def printf(_str, *args):
-    print(_str % args, end='')
-def fprintf(_file, _str, *args):
-    print(_str % args, end='', file=_file)
-def sprintf(_str, *args):
-    return (_str % args)

nilmdb/utils/serializer.py

@@ -1,72 +0,0 @@
-import Queue
-import threading
-import sys
-
-# This file provides a class that will wrap an object and serialize
-# all calls to its methods.  All calls to that object will be queued
-# and executed from a single thread, regardless of which thread makes
-# the call.
-
-# Based partially on http://stackoverflow.com/questions/2642515/
-
-class SerializerThread(threading.Thread):
-    """Thread that retrieves call information from the queue, makes the
-    call, and returns the results."""
-    def __init__(self, call_queue):
-        threading.Thread.__init__(self)
-        self.call_queue = call_queue
-
-    def run(self):
-        while True:
-            result_queue, func, args, kwargs = self.call_queue.get()
-            # Terminate if result_queue is None
-            if result_queue is None:
-                return
-            try:
-                result = func(*args, **kwargs) # wrapped
-            except:
-                result_queue.put((sys.exc_info(), None))
-            else:
-                result_queue.put((None, result))
-
-class WrapCall(object):
-    """Wrap a callable using the given queues"""
-
-    def __init__(self, call_queue, result_queue, func):
-        self.call_queue = call_queue
-        self.result_queue = result_queue
-        self.func = func
-
-    def __call__(self, *args, **kwargs):
-        self.call_queue.put((self.result_queue, self.func, args, kwargs))
-        ( exc_info, result ) = self.result_queue.get()
-        if exc_info is None:
-            return result
-        else:
-            raise exc_info[0], exc_info[1], exc_info[2]
-
-class WrapObject(object):
-    """Wrap all calls to methods in a target object with WrapCall"""
-
-    def __init__(self, target):
-        self.__wrap_target = target
-        self.__wrap_call_queue = Queue.Queue()
-        self.__wrap_serializer = SerializerThread(self.__wrap_call_queue)
-        self.__wrap_serializer.daemon = True
-        self.__wrap_serializer.start()
-
-    def __getattr__(self, key):
-        """Wrap methods of self.__wrap_target in a WrapCall instance"""
-        func = getattr(self.__wrap_target, key)
-        if not callable(func):
-            raise TypeError("Can't serialize attribute %r (type: %s)"
-                            % (key, type(func)))
-        result_queue = Queue.Queue()
-        return WrapCall(self.__wrap_call_queue, result_queue, func)
-
-    def __del__(self):
-        self.__wrap_call_queue.put((None, None, None, None))
-        self.__wrap_serializer.join()
-
-# Just an alias
-Serializer = WrapObject

nilmdb/utils/timer.py

@@ -1,22 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Simple timer to time a block of code, for optimization debugging
-# use like:
-#   with nilmdb.utils.Timer("flush"):
-#       foo.flush()
-
-from __future__ import print_function
-import contextlib
-import time
-
-@contextlib.contextmanager
-def Timer(name = None, tosyslog = False):
-    start = time.time()
-    yield
-    elapsed = int((time.time() - start) * 1000)
-    msg = (name or 'elapsed') + ": " + str(elapsed) + " ms"
-    if tosyslog: # pragma: no cover
-        import syslog
-        syslog.syslog(msg)
-    else:
-        print(msg)

nilmdb/utils/timestamper.py

@@ -1,104 +0,0 @@
-"""File-like objects that add timestamps to the input lines"""
-
-from nilmdb.utils.printf import *
-from nilmdb.utils import datetime_tz
-
-class Timestamper(object):
-    """A file-like object that adds timestamps to lines of an input file."""
-    def __init__(self, infile, ts_iter):
-        """file: filename, or another file-like object
-           ts_iter: iterator that returns a timestamp string for
-           each line of the file"""
-        if isinstance(infile, basestring):
-            self.file = open(infile, "r")
-        else:
-            self.file = infile
-        self.ts_iter = ts_iter
-
-    def close(self):
-        self.file.close()
-
-    def readline(self, *args):
-        while True:
-            line = self.file.readline(*args)
-            if not line:
-                return ""
-            if line[0] == '#':
-                continue
-            break
-        try:
-            return self.ts_iter.next() + line
-        except StopIteration:
-            return ""
-
-    def readlines(self, size = None):
-        out = ""
-        while True:
-            line = self.readline()
-            out += line
-            if not line or (size and len(out) >= size):
-                break
-        return out
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        result = self.readline()
-        if not result:
-            raise StopIteration
-        return result
-
-class TimestamperRate(Timestamper):
-    """Timestamper that uses a start time and a fixed rate"""
-    def __init__(self, infile, start, rate, end = None):
-        """
-        file: file name or object
-
-        start: Unix timestamp for the first value
-
-        rate: 1/rate is added to the timestamp for each line
-
-        end: If specified, raise StopIteration before outputting a value
-             greater than this."""
-        def iterator(start, rate, end):
-            n = 0
-            rate = float(rate)
-            while True:
-                now = start + n / rate
-                if end and now >= end:
-                    raise StopIteration
-                yield sprintf("%.6f ", start + n / rate)
-                n += 1
-        # Handle case where we're passed a datetime or datetime_tz object
-        if "totimestamp" in dir(start):
-            start = start.totimestamp()
-        Timestamper.__init__(self, infile, iterator(start, rate, end))
-        self.start = start
-        self.rate = rate
-    def __str__(self):
-        start = datetime_tz.datetime_tz.fromtimestamp(self.start)
-        start = start.strftime("%a, %d %b %Y %H:%M:%S %Z")
-        return sprintf("TimestamperRate(..., start=\"%s\", rate=%g)",
-                       str(start), self.rate)
-
-class TimestamperNow(Timestamper):
-    """Timestamper that uses current time"""
-    def __init__(self, infile):
-        def iterator():
-            while True:
-                now = datetime_tz.datetime_tz.utcnow().totimestamp()
-                yield sprintf("%.6f ", now)
-        Timestamper.__init__(self, infile, iterator())
-    def __str__(self):
-        return "TimestamperNow(...)"
-
-class TimestamperNull(Timestamper):
-    """Timestamper that adds nothing to each line"""
-    def __init__(self, infile):
-        def iterator():
-            while True:
-                yield ""
-        Timestamper.__init__(self, infile, iterator())
-    def __str__(self):
-        return "TimestamperNull(...)"

nilmdb/utils/urllib.py

@@ -1,37 +0,0 @@
-from __future__ import absolute_import
-from urllib import quote_plus, _is_unicode
-
-# urllib.urlencode insists on encoding Unicode as ASCII.  This is based
-# on that function, except we always encode it as UTF-8 instead.
-
-def urlencode(query):
-    """Encode a dictionary into a URL query string.
-
-    If any values in the query arg are sequences, each sequence
-    element is converted to a separate parameter.
-    """
-
-    query = query.items()
-
-    l = []
-    for k, v in query:
-        k = quote_plus(str(k))
-        if isinstance(v, str):
-            v = quote_plus(v)
-            l.append(k + '=' + v)
-        elif _is_unicode(v):
-            v = quote_plus(v.encode("utf-8","strict"))
-            l.append(k + '=' + v)
-        else:
-            try:
-                # is this a sufficient test for sequence-ness?
-                len(v)
-            except TypeError:
-                # not a sequence
-                v = quote_plus(str(v))
-                l.append(k + '=' + v)
-            else:
-                # loop over the sequence
-                for elt in v:
-                    l.append(k + '=' + quote_plus(str(elt)))
-    return '&'.join(l)

pytables-test/Makefile

@@ -0,0 +1,5 @@
+all:
+	time python test-indexed-read.py
+
+clean:
+	rm -f *pyc

pytables-test/README.jim

@@ -0,0 +1,2 @@
+New version from:
+  http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=641485#15

pytables-test/TODO.txt

@@ -0,0 +1,12 @@
+- Make CherryPy server that can handle simple GET/POST,
+  and a httplib client that can talk to that server.
+  Steps:
+  - Make server handle GET
+  - Make client send request, get response
+  - Add request streaming to server
+  - Add request streaming to client
+  - Make server handle POST
+  - Make client send request, get response
+  - Add request streaming to server
+  - Add request streaming to client
+  - Integrate into a server process that also keeps database open.

pytables-test/indexing-notes

@@ -0,0 +1,3 @@
+Indexing time64 doesn't seem to work -- needed to do "time >= 1243052015" even though the actual database times
+should be something like 1243052015.847000.  Let's switch to just using a 64-bit integer counting e.g.
+microseconds since 1970-01-01 

pytables-test/sample-query

@@ -0,0 +1,3 @@
+timestamp > 1243052015
+took 394.5 minutes in vitables
+(2340 rows matched)

pytables-test/server.py

@@ -0,0 +1,53 @@
+import sys
+import tables
+import nilmdb
+
+try:
+    import cherrypy
+    cherrypy.tools.json_out
+except:
+    sys.stderr.write("Cherrypy 3.2+ required\n")
+    sys.exit(1)
+
+class NilmApp:
+    def __init__(self, db):
+        self.db = db
+
+class Root(NilmApp):
+    """NILM Database"""
+
+    server_version = "1.0"
+
+    @cherrypy.expose
+    def index(self):
+        raise cherrypy.NotFound()
+
+    @cherrypy.expose
+    def favicon_ico(self):
+        raise cherrypy.NotFound()
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def version(self):
+        return self.server_version
+
+class Stream(NilmApp):
+    """Stream operations"""
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def list(self):
+        return 
+    
+cherrypy.config.update({
+    'server.socket_host': '127.0.0.1',
+    'server.socket_port': 12380
+    })
+
+db = nilmdb.nilmdb()
+cherrypy.tree.mount(Root(db), "/")
+cherrypy.tree.mount(Stream(db), "/stream")
+
+if __name__ == "__main__":
+    cherrypy.engine.start()
+    cherrypy.engine.block()

pytables-test/speed-pytables.py

@@ -0,0 +1,16 @@
+import tables
+import numpy
+
+class RawSample(tables.IsDescription):
+    timestamp = tables.UInt64Col()
+    voltage   = tables.UInt16Col(shape = 3)
+    current   = tables.UInt16Col(shape = 3)
+
+h5file = tables.openFile("test.h5", mode = "w", title = "Test")
+group = h5file.createGroup("/", "raw", "Raw Data")
+table = h5file.createTable(group, "nilm1", RawSample, "NILM 1")
+
+print repr(h5file)
+
+# write rows
+

pytables-test/test-indexing.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+from tables import *
+import re
+import time
+
+# A class to describe our data
+class PrepData(IsDescription):
+    timestamp = Int64Col()
+    p1 = Float32Col()
+    q1 = Float32Col()
+    p3 = Float32Col()
+    q3 = Float32Col()
+    p5 = Float32Col()
+    q5 = Float32Col()
+    p7 = Float32Col()
+    q7 = Float32Col()
+
+filename = "test.h5"
+h5file = openFile(filename, mode = "w", title = "NILM Test")
+
+group = h5file.createGroup("/", "newton", "Newton school")
+table = h5file.createTable(group, "prep", PrepData, "Prep Data", expectedrows = 120 * 86400 * 90)
+
+table.cols.timestamp.createIndex()
+
+for i in range(0, 80):
+    # Open file
+    data = open("data/alldata")
+    count = 0
+    oldtime = time.time()
+    prep = table.row
+    for line in data:
+        count = count + 1
+        if count % 1000000 == 0:
+            print str(i) + ": " + str((time.time() - oldtime)) + ", total " + str(count/1000000) + "m lines"
+            oldtime = time.time()
+        v = re.split('\s+', line)
+        prep['timestamp'] = int(v[0]) + 500000000 * i
+        prep['p1'] = v[1]
+        prep['q1'] = v[2]
+        prep['p3'] = v[3]
+        prep['q3'] = v[4]
+        prep['p5'] = v[5]
+        prep['q5'] = v[6]
+        prep['p7'] = v[7]
+        prep['q7'] = v[8]
+        prep.append()
+    data.close()
+
+h5file.close()
+
+
+

pytables-test/test-write.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+from tables import *
+import re
+import time
+
+# A class to describe our data
+class PrepData(IsDescription):
+    timestamp = Time64Col()
+    p1 = Float32Col()
+    q1 = Float32Col()
+    p3 = Float32Col()
+    q3 = Float32Col()
+    p5 = Float32Col()
+    q5 = Float32Col()
+    p7 = Float32Col()
+    q7 = Float32Col()
+
+filename = "test.h5"
+h5file = openFile(filename, mode = "w", title = "NILM Test")
+
+group = h5file.createGroup("/", "newton", "Newton school")
+table = h5file.createTable(group, "prep", PrepData, "Prep Data")
+
+table.cols.timestamp.createIndex()
+
+for i in range(0, 80):
+    # Open file
+    data = open("data/alldata")
+    count = 0
+    oldtime = time.time()
+    prep = table.row
+    for line in data:
+        count = count + 1
+        if count % 1000000 == 0:
+            print str(i) + ": " + str((time.time() - oldtime)) + ", total " + str(count/1000000) + "m lines"
+            oldtime = time.time()
+        v = re.split('\s+', line)
+        prep['timestamp'] = float(v[0]) / 1000.0 + 500000 * i
+        prep['p1'] = v[1]
+        prep['q1'] = v[2]
+        prep['p3'] = v[3]
+        prep['q3'] = v[4]
+        prep['p5'] = v[5]
+        prep['q5'] = v[6]
+        prep['p7'] = v[7]
+        prep['q7'] = v[8]
+        prep.append()
+    data.close()
+
+h5file.close()
+
+
+

setup.cfg

@@ -1,40 +1,9 @@
-[aliases]
-test = nosetests
-
 [nosetests]
-# Note: values must be set to 1, and have no comments on the same line,
-# for "python setup.py nosetests" to work correctly.
 nocapture=1
-# Comment this out to see CherryPy logs on failure:
-nologcapture=1
 with-coverage=1
 cover-inclusive=1
 cover-package=nilmdb
 cover-erase=1
-# this works, puts html output in cover/ dir:
-# cover-html=1
-# need nose 1.1.3 for this:
-# cover-branches=1
-#debug=nose
-#debug-log=nose.log
 stop=1
 verbosity=2
-tests=tests
-#tests=tests/test_bulkdata.py
-#tests=tests/test_mustclose.py
-#tests=tests/test_lrucache.py
-#tests=tests/test_cmdline.py
-#tests=tests/test_layout.py
-#tests=tests/test_rbtree.py
-#tests=tests/test_interval.py
-#tests=tests/test_rbtree.py,tests/test_interval.py
-#tests=tests/test_interval.py
-#tests=tests/test_client.py
-#tests=tests/test_timestamper.py
-#tests=tests/test_serializer.py
-#tests=tests/test_iteratorizer.py
-#tests=tests/test_client.py:TestClient.test_client_nilmdb
-#tests=tests/test_nilmdb.py
-#with-profile=1
-#profile-sort=time
-##profile-restrict=10  # doesn't work right, treated as string or something
+

setup.py

@@ -1,135 +1,9 @@
 #!/usr/bin/python
 
-# To release a new version, tag it:
-#   git tag -a nilmdb-1.1 -m "Version 1.1"
-#   git push --tags
-# Then just package it up:
-#   python setup.py sdist
+from distutils.core import setup
 
-# This is supposed to be using Distribute:
-#
-#   distutils provides a "setup" method.
-#   setuptools is a set of monkeypatches on top of that.
-#   distribute is a particular version/implementation of setuptools.
-#
-# So we don't really know if this is using the old setuptools or the
-# Distribute-provided version of setuptools.
-
-import traceback
-import sys
-import os
-
-try:
-    from setuptools import setup, find_packages
-    from distutils.extension import Extension
-    import distutils.version
-except ImportError:
-    traceback.print_exc()
-    print "Please install the prerequisites listed in README.txt"
-    sys.exit(1)
-
-# Versioneer manages version numbers from git tags.
-# https://github.com/warner/python-versioneer
-import versioneer
-versioneer.versionfile_source = 'nilmdb/_version.py'
-versioneer.versionfile_build = 'nilmdb/_version.py'
-versioneer.tag_prefix = 'nilmdb-'
-versioneer.parentdir_prefix = 'nilmdb-'
-
-# Hack to workaround logging/multiprocessing issue:
-# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
-try: import multiprocessing
-except: pass
-
-# Use Cython if it's new enough, otherwise use preexisting C files.
-cython_modules = [ 'nilmdb.server.interval',
-                   'nilmdb.server.layout',
-                   'nilmdb.server.rbtree' ]
-try:
-    import Cython
-    from Cython.Build import cythonize
-    if (distutils.version.LooseVersion(Cython.__version__) <
-        distutils.version.LooseVersion("0.16")):
-        print "Cython version", Cython.__version__, "is too old; not using it."
-        raise ImportError()
-    use_cython = True
-except ImportError:
-    use_cython = False
-
-ext_modules = []
-for modulename in cython_modules:
-    filename = modulename.replace('.','/')
-    if use_cython:
-        ext_modules.extend(cythonize(filename + ".pyx"))
-    else:
-        cfile = filename + ".c"
-        if not os.path.exists(cfile):
-            raise Exception("Missing source file " + cfile + ".  "
-                            "Try installing cython >= 0.16.")
-        ext_modules.append(Extension(modulename, [ cfile ]))
-
-# We need a MANIFEST.in.  Generate it here rather than polluting the
-# repository with yet another setup-related file.
-with open("MANIFEST.in", "w") as m:
-    m.write("""
-# Root
-include README.txt
-include setup.cfg
-include setup.py
-include versioneer.py
-include Makefile
-include .coveragerc
-include .pylintrc
-
-# Cython files -- include source.
-recursive-include nilmdb/server *.pyx *.pyxdep *.pxd
-
-# Tests
-recursive-include tests *.py
-recursive-include tests/data *
-include tests/test.order
-
-# Docs
-recursive-include docs Makefile *.md
-""")
-
-# Run setup
-setup(name='nilmdb',
-      version = versioneer.get_version(),
-      cmdclass = versioneer.get_cmdclass(),
-      url = 'https://git.jim.sh/jim/lees/nilmdb.git',
-      author = 'Jim Paris',
-      description = "NILM Database",
-      long_description = "NILM Database",
-      license = "Proprietary",
-      author_email = 'jim@jtan.com',
-      tests_require = [ 'nose',
-                        'coverage',
-                        ],
-      setup_requires = [ 'distribute',
-                         ],
-      install_requires = [ 'decorator',
-                           'cherrypy >= 3.2',
-                           'simplejson',
-                           'pycurl',
-                           'python-dateutil',
-                           'pytz',
-                           'psutil >= 0.3.0',
-                           ],
-      packages = [ 'nilmdb',
-                   'nilmdb.utils',
-                   'nilmdb.utils.datetime_tz',
-                   'nilmdb.server',
-                   'nilmdb.client',
-                   'nilmdb.cmdline',
-                   'nilmdb.scripts',
-                   ],
-      entry_points = {
-          'console_scripts': [
-              'nilmtool = nilmdb.scripts.nilmtool:main',
-              'nilmdb-server = nilmdb.scripts.nilmdb_server:main',
-              ],
-          },
-      ext_modules = ext_modules,
-      zip_safe = False,
+setup(name = 'nilmdb',
+      version = '1.0',
+      scripts = [ 'bin/nilm-test.py' ],
+      packages = [ 'nilmdb' ],
       )

test/Makefile

@@ -0,0 +1,5 @@
+all:
+	python speed-readascii.py
+
+clean:
+	rm -f *pyc

test/printf.py

@@ -0,0 +1,4 @@
+from __future__ import print_function
+def printf(str, *args):
+    print(str % args, end='')
+    

test/speed-readascii.py

@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+from printf import printf
+import time
+import re
+import numpy as np
+import itertools
+
+class Timer():
+    def __init__(self, arg):
+        self.arg = arg
+    def __enter__(self): self.start = time.time()
+    def __exit__(self, *args): printf("%s: %f lines/sec\n", self.arg, 1e6 / (time.time() - self.start))
+
+def test_split():
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [0]*6
+        tmp = [ int(i) for i in line.partition('#')[0].split() ]
+        out[0:len(tmp)] = tmp
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_split2():
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [0]*6
+        tmp = [ int(i,10) for i in line.partition('#')[0].split() ]
+        out[0:len(tmp)] = tmp
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_regex():
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [0]*6
+        tmp = [ int(x) for x in re.findall('(\d+)\s+',line.partition('#')[0]) ]
+        out[0:len(tmp)] = tmp
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_bigregex():
+    regex = re.compile('^(?:\s*)' + '(?:(\d+)\s+)?' * 6)
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [ int(x or 0) for x in re.match(regex, line).groups() ]
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_numpy():
+    out = np.genfromtxt(open('1m.raw', 'r'),
+                        dtype = np.dtype('i2,i2,i2,i2,i2,i2'))
+    
+with Timer("numpy"):
+    test_numpy() # 106k/sec
+
+with Timer("regex"):
+    test_regex() # 121k/sec
+
+with Timer("split"):
+    test_split() # 219k/sec
+
+with Timer("split2"):
+    test_split2() # 328k/sec
+
+with Timer("bigregex"):
+    test_bigregex() # 130k/sec
+
+# The "int" operation takes quite a while -- int(x,10) is twice as fast
+# Perl does about 500k/sec
+

test/speed-readbinary.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+
+from printf import printf
+import time
+import re
+import numpy as np
+import itertools
+import struct
+import array
+import os
+import mmap
+
+class Timer():
+    def __init__(self, arg):
+        self.arg = arg
+    def __enter__(self): self.start = time.time()
+    def __exit__(self, *args): printf("%s: %f klines/sec\n", self.arg, 1e3 / (time.time() - self.start))
+
+def test_struct1():
+    """read with struct.unpack"""
+    f = open('1m.bin', 'rb')
+    f.seek(0,os.SEEK_END)
+    filesize = f.tell()
+    f.seek(0,os.SEEK_SET)
+    packer = struct.Struct('!dHHHHHH')
+    items = filesize / packer.size
+    for n in xrange(items):
+        s = f.read(packer.size)
+        out = packer.unpack(s)
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_struct2():
+    """read with struct.unpack, convert to string"""
+    f = open('1m.bin', 'rb')
+    f.seek(0,os.SEEK_END)
+    filesize = f.tell()
+    f.seek(0,os.SEEK_SET)
+    packer = struct.Struct('!dHHHHHH')
+    items = filesize / packer.size
+    for n in xrange(items):
+        s = f.read(packer.size)
+        out = packer.unpack(s)
+        x = str(out)
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_mmap():
+    """struct.unpack with mmap"""
+    with open('1m.bin', 'rb') as f:
+        f.seek(0,os.SEEK_END)
+        filesize = f.tell()
+        f.seek(0,os.SEEK_SET)
+        m = mmap.mmap(f.fileno(), filesize, access=mmap.ACCESS_READ)
+        packer = struct.Struct('!dHHHHHH')
+        items = filesize / packer.size
+        for n in xrange(items):
+            out = packer.unpack(m[packer.size*n : packer.size*(n+1)])
+            if (n % 100000 == 0):
+                printf("line %d = %s\n", n, str(out))
+
+with Timer("mmap"):
+    test_mmap()  # 1600k
+
+with Timer("struct1"):
+    test_struct1()  # 1460k
+
+with Timer("struct2"):
+    test_struct2()  # 210k
+
+# Reading from the file is again much quicker than converting to string
+# Use mmap, it's good
+
+

test/speed-writebinary.py

@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+from printf import printf
+import time
+import re
+import numpy as np
+import itertools
+import struct
+import array
+
+class Timer():
+    def __init__(self, arg):
+        self.arg = arg
+    def __enter__(self): self.start = time.time()
+    def __exit__(self, *args): printf("%s: %f klines/sec\n", self.arg, 1e3 / (time.time() - self.start))
+
+def read_ascii():
+    for n in xrange(1000000):
+        yield (1234, 2345, 3456, 4576, 5678, 6789)
+#    for n, line in enumerate(open('1m.raw', 'r')):
+#        out = [0]*6
+#        tmp = [ int(i,10) for i in line.partition('#')[0].split() ]
+#        out[0:len(tmp)] = tmp
+#        if (n % 100000 == 0):
+#            printf("line %d = %s\n", n, str(out))
+#        yield out
+
+def test_struct1():
+    """write with struct.pack"""
+    f = open('1m.bin', 'wb')
+    for out in read_ascii():
+        s = struct.pack('!HHHHHH', *out)
+        f.write(s)
+
+def test_struct2():
+    """use constant format string"""
+    f = open('1m.bin', 'wb')
+    packer = struct.Struct('!HHHHHH')
+    for out in read_ascii():
+        f.write(packer.pack(*out))
+    f.close()
+    printf("size was %d\n", packer.size)
+
+def test_struct3():
+    """like struct1, with timestamp"""
+    f = open('1m.bin', 'wb')
+    for out in read_ascii():
+        s = struct.pack('!dHHHHHH', time.time(), *out)
+        f.write(s)
+
+def test_struct4():
+    """like struct2, with timestamp"""
+    f = open('1m.bin', 'wb')
+    packer = struct.Struct('!dHHHHHH')
+    for out in read_ascii():
+        f.write(packer.pack(time.time(), *out))
+    f.close()
+    printf("size was %d\n", packer.size)
+
+#raise Exception('done')
+
+with Timer("struct1"):
+    test_struct1() # 1089k
+
+with Timer("struct2"):
+    test_struct2() # 1249k
+
+with Timer("struct3"):
+    test_struct3() # 845k
+
+with Timer("struct4"):
+    test_struct4() # 922k
+
+# This seems fast enough for writing new data, since it's faster than
+# we read ascii data anyway.  Use e.g. struct4
+

test/test-struct-pack.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+import struct
+import mmap
+
+f = open("test.dat", "rb+")
+mm = mmap.mmap(f.fileno(),3)
+
+print len(mm)
+print "first 3 bytes: " + mm[0:3];
+

test/test.dat

@@ -0,0 +1 @@
+

















test/todo.md

@@ -0,0 +1,7 @@
+- Have a class representing the file contents
+- Looks like an array
+  - len(), get(), index
+  - some form of bisect search
+  - get_extents = return [0].timestamp, [-1].timestamp
+  - 
+- Can append?  Sure, why not.  Just write to the file, extend mmap accordingly.

tests/data/extract-1

@@ -1,124 +0,0 @@
-# path: /newton/prep
-# layout: PrepData
-# start: Fri, 23 Mar 2012 10:00:30.000000 +0000
-# end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010

tests/data/extract-2

@@ -1,119 +0,0 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010

tests/data/extract-3

@@ -1 +0,0 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883

tests/data/extract-4

@@ -1,2 +0,0 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039

tests/data/extract-5

@@ -1,124 +0,0 @@
-# path: /newton/prep
-# layout: PrepData
-# start: Fri, 23 Mar 2012 10:00:30.000000 +0000
-# end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010

tests/data/extract-6

@@ -1,120 +0,0 @@
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010

tests/data/extract-7

@@ -1,124 +0,0 @@
-# path: /newton/prep
-# layout: PrepData
-# start: 1332496830.0
-# end: 1332496830.999
-1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010

tests/data/prep-20120323T1002-first19lines

@@ -1,19 +0,0 @@
-2.56437e+05  2.24430e+05  4.01161e+03  3.47534e+03  7.49589e+03  3.38894e+03  2.61397e+02  3.73126e+03  
-2.53963e+05  2.24167e+05  5.62107e+03  1.54801e+03  9.16517e+03  3.52293e+03  1.05893e+03  2.99696e+03  
-2.58508e+05  2.24930e+05  6.01140e+03  8.18866e+02  9.03995e+03  4.48244e+03  2.49039e+03  2.67934e+03  
-2.59627e+05  2.26022e+05  4.47450e+03  2.42302e+03  7.41419e+03  5.07197e+03  2.43938e+03  2.96296e+03  
-2.55187e+05  2.24632e+05  4.73857e+03  3.39804e+03  7.39512e+03  4.72645e+03  1.83903e+03  3.39353e+03  
-2.57102e+05  2.21623e+05  6.14413e+03  1.44109e+03  8.75648e+03  3.49532e+03  1.86994e+03  3.75253e+03  
-2.63653e+05  2.21770e+05  6.22177e+03  7.38962e+02  9.54760e+03  2.66682e+03  1.46266e+03  3.33257e+03  
-2.63613e+05  2.25256e+05  4.47712e+03  2.43745e+03  8.51021e+03  3.85563e+03  9.59442e+02  2.38718e+03  
-2.55350e+05  2.26264e+05  4.28372e+03  3.92394e+03  7.91247e+03  5.46652e+03  1.28499e+03  2.09372e+03  
-2.52727e+05  2.24609e+05  5.85193e+03  2.49198e+03  8.54063e+03  5.62305e+03  2.33978e+03  3.00714e+03  
-2.58475e+05  2.23578e+05  5.92487e+03  1.39448e+03  8.77962e+03  4.54418e+03  2.13203e+03  3.84976e+03  
-2.61563e+05  2.24609e+05  4.33614e+03  2.45575e+03  8.05538e+03  3.46911e+03  6.27873e+02  3.66420e+03  
-2.56401e+05  2.24441e+05  4.18715e+03  3.45717e+03  7.90669e+03  3.53355e+03  -5.84482e+00  2.96687e+03  
-2.54745e+05  2.22644e+05  6.02005e+03  1.94721e+03  9.28939e+03  3.80020e+03  1.34820e+03  2.37785e+03  
-2.60723e+05  2.22660e+05  6.69719e+03  1.03048e+03  9.26124e+03  4.34917e+03  2.84530e+03  2.73619e+03  
-2.63089e+05  2.25711e+05  4.77887e+03  2.60417e+03  7.39660e+03  4.59811e+03  2.17472e+03  3.40729e+03  
-2.55843e+05  2.27128e+05  4.02413e+03  4.39323e+03  6.79336e+03  4.62535e+03  7.52009e+02  3.44647e+03  
-2.51904e+05  2.24868e+05  5.82289e+03  3.02127e+03  8.46160e+03  3.80298e+03  8.07212e+02  3.53468e+03  
-2.57670e+05  2.22974e+05  6.73436e+03  1.60956e+03  9.92960e+03  2.98028e+03  1.44168e+03  3.05351e+03  

tests/data/prep-20120323T1004-badtimes

@@ -1,11 +0,0 @@
-1332497040.000000 2.56439e+05  2.24775e+05  2.92897e+03  4.66646e+03  7.58491e+03  3.57351e+03  -4.34171e+02  2.98819e+03  
-1332497040.010000 2.51903e+05  2.23202e+05  4.23696e+03  3.49363e+03  8.53493e+03  4.29416e+03  8.49573e+02  2.38189e+03  
-1332497040.020000 2.57625e+05  2.20247e+05  5.47017e+03  1.35872e+03  9.18903e+03  4.56136e+03  2.65599e+03  2.60912e+03  
-1332497040.030000 2.63375e+05  2.20706e+05  4.51842e+03  1.80758e+03  8.17208e+03  4.17463e+03  2.57884e+03  3.32848e+03  
-1332497040.040000 2.59221e+05  2.22346e+05  2.98879e+03  3.66264e+03  6.87274e+03  3.94223e+03  1.25928e+03  3.51786e+03  
-1332497040.050000 2.51918e+05  2.22281e+05  4.22677e+03  2.84764e+03  7.78323e+03  3.81659e+03  8.04944e+02  3.46314e+03  
-1332497040.050000 2.54478e+05  2.21701e+05  5.61366e+03  1.02262e+03  9.26581e+03  3.50152e+03  1.29331e+03  3.07271e+03  
-1332497040.060000 2.59568e+05  2.22945e+05  4.97190e+03  1.28250e+03  8.62081e+03  4.06316e+03  1.85717e+03  2.61990e+03  
-1332497040.070000 2.57269e+05  2.23697e+05  3.60527e+03  3.05749e+03  7.22363e+03  4.90330e+03  1.93736e+03  2.35357e+03  
-1332497040.080000 2.52274e+05  2.21438e+05  5.01228e+03  2.86309e+03  7.87115e+03  4.80448e+03  2.18291e+03  2.93397e+03  
-1332497040.090000 2.56468e+05  2.19205e+05  6.29804e+03  8.09467e+02  9.12895e+03  3.52055e+03  2.16980e+03  3.88739e+03  

tests/runtests.py

@@ -1,49 +0,0 @@
-#!/usr/bin/python
-
-import nose
-import os
-import sys
-import glob
-from collections import OrderedDict
-
-# Change into parent dir
-os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/..")
-
-class JimOrderPlugin(nose.plugins.Plugin):
-    """When searching for tests and encountering a directory that
-    contains a 'test.order' file, run tests listed in that file, in the
-    order that they're listed.  Globs are OK in that file and duplicates
-    are removed."""
-    name = 'jimorder'
-    score = 10000
-
-    def prepareTestLoader(self, loader):
-        def wrap(func):
-            def wrapper(name, *args, **kwargs):
-                addr = nose.selector.TestAddress(
-                    name, workingDir=loader.workingDir)
-                try:
-                    order = os.path.join(addr.filename, "test.order")
-                except:
-                    order = None
-                if order and os.path.exists(order):
-                    files = []
-                    for line in open(order):
-                        line = line.split('#')[0].strip()
-                        if not line:
-                            continue
-                        fn = os.path.join(addr.filename, line.strip())
-                        files.extend(sorted(glob.glob(fn)) or [fn])
-                    files = list(OrderedDict.fromkeys(files))
-                    tests = [ wrapper(fn, *args, **kwargs) for fn in files ]
-                    return loader.suiteClass(tests)
-                return func(name, *args, **kwargs)
-            return wrapper
-        loader.loadTestsFromName = wrap(loader.loadTestsFromName)
-        return loader
-
-# Use setup.cfg for most of the test configuration.  Adding
-# --with-jimorder here means that a normal "nosetests" run will
-# still work, it just won't support test.order.
-nose.main(addplugins = [ JimOrderPlugin() ],
-          argv = sys.argv + ["--with-jimorder"])

tests/test.order

@@ -1,18 +0,0 @@
-test_printf.py
-test_lrucache.py
-test_mustclose.py
-
-test_serializer.py
-test_iteratorizer.py
-
-test_timestamper.py
-test_layout.py
-test_rbtree.py
-test_interval.py
-
-test_bulkdata.py
-test_nilmdb.py
-test_client.py
-test_cmdline.py
-
-test_*.py

tests/test_bulkdata.py

@@ -1,102 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import nilmdb
-from nilmdb.utils.printf import *
-from nose.tools import *
-from nose.tools import assert_raises
-import itertools
-
-from testutil.helpers import *
-
-testdb = "tests/bulkdata-testdb"
-
-import nilmdb.server.bulkdata
-from nilmdb.server.bulkdata import BulkData
-
-class TestBulkData(object):
-
-    def test_bulkdata(self):
-        for (size, files, db) in [ ( 0, 0, testdb ),
-                                       ( 25, 1000, testdb ),
-                                       ( 1000, 3, testdb.decode("utf-8") ) ]:
-            recursive_unlink(db)
-            os.mkdir(db)
-            self.do_basic(db, size, files)
-
-    def do_basic(self, db, size, files):
-        """Do the basic test with variable file_size and files_per_dir"""
-        if not size or not files:
-            data = BulkData(db)
-        else:
-            data = BulkData(db, file_size = size, files_per_dir = files)
-
-        # create empty
-        with assert_raises(ValueError):
-            data.create("/foo", "uint16_8")
-        with assert_raises(ValueError):
-            data.create("foo/bar", "uint16_8")
-        with assert_raises(ValueError):
-            data.create("/foo/bar", "uint8_8")
-        data.create("/foo/bar", "uint16_8")
-        data.create(u"/foo/baz/quux", "float64_16")
-        with assert_raises(ValueError):
-            data.create("/foo/bar/baz", "uint16_8")
-        with assert_raises(ValueError):
-            data.create("/foo/baz", "float64_16")
-
-        # get node -- see if caching works
-        nodes = []
-        for i in range(5000):
-            nodes.append(data.getnode("/foo/bar"))
-            nodes.append(data.getnode("/foo/baz/quux"))
-        del nodes
-
-        # Test node
-        node = data.getnode("/foo/bar")
-        with assert_raises(IndexError):
-            x = node[0]
-        raw = []
-        for i in range(1000):
-            raw.append([10000+i, 1, 2, 3, 4, 5, 6, 7, 8 ])
-        node.append(raw[0:1])
-        node.append(raw[1:100])
-        node.append(raw[100:])
-
-        misc_slices = [ 0, 100, slice(None), slice(0), slice(10),
-                        slice(5,10), slice(3,None), slice(3,-3),
-                        slice(20,10), slice(200,100,-1), slice(None,0,-1),
-                        slice(100,500,5) ]
-        # Extract slices
-        for s in misc_slices:
-            eq_(node[s], raw[s])
-
-        # Get some coverage of remove; remove is more fully tested
-        # in cmdline
-        with assert_raises(IndexError):
-            node.remove(9999,9998)
-
-        # close, reopen
-        # reopen
-        data.close()
-        if not size or not files:
-            data = BulkData(db)
-        else:
-            data = BulkData(db, file_size = size, files_per_dir = files)
-        node = data.getnode("/foo/bar")
-
-        # Extract slices
-        for s in misc_slices:
-            eq_(node[s], raw[s])
-
-        # destroy
-        with assert_raises(ValueError):
-            data.destroy("/foo")
-        with assert_raises(ValueError):
-            data.destroy("/foo/baz")
-        with assert_raises(ValueError):
-            data.destroy("/foo/qwerty")
-        data.destroy("/foo/baz/quux")
-        data.destroy("/foo/bar")
-
-        # close
-        data.close()

tests/test_client.py

@@ -1,578 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import nilmdb
-from nilmdb.utils.printf import *
-from nilmdb.utils import timestamper
-from nilmdb.client import ClientError, ServerError
-from nilmdb.utils import datetime_tz
-
-from nose.tools import *
-from nose.tools import assert_raises
-import itertools
-import distutils.version
-import os
-import sys
-import threading
-import cStringIO
-import simplejson as json
-import unittest
-import warnings
-import resource
-import time
-
-from testutil.helpers import *
-
-testdb = "tests/client-testdb"
-testurl = "http://localhost:12380/"
-
-def setup_module():
-    global test_server, test_db
-    # Clear out DB
-    recursive_unlink(testdb)
-
-    # Start web app on a custom port
-    test_db = nilmdb.NilmDB(testdb, sync = False)
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
-                                fast_shutdown = True,
-                                force_traceback = False)
-    test_server.start(blocking = False)
-
-def teardown_module():
-    global test_server, test_db
-    # Close web app
-    test_server.stop()
-    test_db.close()
-
-class TestClient(object):
-
-    def test_client_01_basic(self):
-        # Test a fake host
-        client = nilmdb.Client(url = "http://localhost:1/")
-        with assert_raises(nilmdb.client.ServerError):
-            client.version()
-        client.close()
-
-        # Trigger same error with a PUT request
-        client = nilmdb.Client(url = "http://localhost:1/")
-        with assert_raises(nilmdb.client.ServerError):
-            client.version()
-        client.close()
-
-        # Then a fake URL on a real host
-        client = nilmdb.Client(url = "http://localhost:12380/fake/")
-        with assert_raises(nilmdb.client.ClientError):
-            client.version()
-        client.close()
-
-        # Now a real URL with no http:// prefix
-        client = nilmdb.Client(url = "localhost:12380")
-        version = client.version()
-        client.close()
-
-        # Now use the real URL
-        client = nilmdb.Client(url = testurl)
-        version = client.version()
-        eq_(distutils.version.LooseVersion(version),
-            distutils.version.LooseVersion(test_server.version))
-
-        # Bad URLs should give 404, not 500
-        with assert_raises(ClientError):
-            client.http.get("/stream/create")
-        client.close()
-
-    def test_client_02_createlist(self):
-        # Basic stream tests, like those in test_nilmdb:test_stream
-        client = nilmdb.Client(url = testurl)
-
-        # Database starts empty
-        eq_(client.stream_list(), [])
-
-        # Bad path
-        with assert_raises(ClientError):
-            client.stream_create("foo/bar/baz", "PrepData")
-        with assert_raises(ClientError):
-            client.stream_create("/foo", "PrepData")
-        # Bad layout type
-        with assert_raises(ClientError):
-            client.stream_create("/newton/prep", "NoSuchLayout")
-
-        # Create three streams
-        client.stream_create("/newton/prep", "PrepData")
-        client.stream_create("/newton/raw", "RawData")
-        client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
-
-        # Verify we got 3 streams
-        eq_(client.stream_list(), [ ["/newton/prep", "PrepData"],
-                                    ["/newton/raw", "RawData"],
-                                    ["/newton/zzz/rawnotch", "RawNotchedData"]
-                                    ])
-        # Match just one type or one path
-        eq_(client.stream_list(layout="RawData"),
-            [ ["/newton/raw", "RawData"] ])
-        eq_(client.stream_list(path="/newton/raw"),
-            [ ["/newton/raw", "RawData"] ])
-
-        # Try messing with resource limits to trigger errors and get
-        # more coverage.  Here, make it so we can only create files 1
-        # byte in size, which will trigger an IOError in the server when
-        # we create a table.
-        limit = resource.getrlimit(resource.RLIMIT_FSIZE)
-        resource.setrlimit(resource.RLIMIT_FSIZE, (1, limit[1]))
-        with assert_raises(ServerError) as e:
-            client.stream_create("/newton/hello", "RawData")
-        resource.setrlimit(resource.RLIMIT_FSIZE, limit)
-
-        client.close()
-
-    def test_client_03_metadata(self):
-        client = nilmdb.Client(url = testurl)
-
-        # Set / get metadata
-        eq_(client.stream_get_metadata("/newton/prep"), {})
-        eq_(client.stream_get_metadata("/newton/raw"), {})
-        meta1 = { "description": "The Data",
-                  "v_scale": "1.234" }
-        meta2 = { "description": "The Data" }
-        meta3 = { "v_scale": "1.234" }
-        client.stream_set_metadata("/newton/prep", meta1)
-        client.stream_update_metadata("/newton/prep", {})
-        client.stream_update_metadata("/newton/raw", meta2)
-        client.stream_update_metadata("/newton/raw", meta3)
-        eq_(client.stream_get_metadata("/newton/prep"), meta1)
-        eq_(client.stream_get_metadata("/newton/raw"), meta1)
-        eq_(client.stream_get_metadata("/newton/raw",
-                                       [ "description" ] ), meta2)
-        eq_(client.stream_get_metadata("/newton/raw",
-                                       [ "description", "v_scale" ] ), meta1)
-
-        # missing key
-        eq_(client.stream_get_metadata("/newton/raw", "descr"),
-            { "descr": None })
-        eq_(client.stream_get_metadata("/newton/raw", [ "descr" ]),
-            { "descr": None })
-
-        # test wrong types (list instead of dict)
-        with assert_raises(ClientError):
-            client.stream_set_metadata("/newton/prep", [1,2,3])
-        with assert_raises(ClientError):
-            client.stream_update_metadata("/newton/prep", [1,2,3])
-        client.close()
-
-    def test_client_04_insert(self):
-        client = nilmdb.Client(url = testurl)
-
-        datetime_tz.localtz_set("America/New_York")
-
-        testfile = "tests/data/prep-20120323T1000"
-        start = datetime_tz.datetime_tz.smartparse("20120323T1000")
-        start = start.totimestamp()
-        rate = 120
-
-        # First try a nonexistent path
-        data = timestamper.TimestamperRate(testfile, start, 120)
-        with assert_raises(ClientError) as e:
-            result = client.stream_insert("/newton/no-such-path", data)
-        in_("404 Not Found", str(e.exception))
-
-        # Now try reversed timestamps
-        data = timestamper.TimestamperRate(testfile, start, 120)
-        data = reversed(list(data))
-        with assert_raises(ClientError) as e:
-            result = client.stream_insert("/newton/prep", data)
-        in_("400 Bad Request", str(e.exception))
-        in_("timestamp is not monotonically increasing", str(e.exception))
-
-        # Now try empty data (no server request made)
-        empty = cStringIO.StringIO("")
-        data = timestamper.TimestamperRate(empty, start, 120)
-        result = client.stream_insert("/newton/prep", data)
-        eq_(result, None)
-
-        # It's OK to insert an empty interval
-        client.http.put("stream/insert", "", { "path": "/newton/prep",
-                                               "start": 1, "end": 2 })
-        eq_(list(client.stream_intervals("/newton/prep")), [[1, 2]])
-        client.stream_remove("/newton/prep")
-        eq_(list(client.stream_intervals("/newton/prep")), [])
-
-        # Timestamps can be negative too
-        client.http.put("stream/insert", "", { "path": "/newton/prep",
-                                               "start": -2, "end": -1 })
-        eq_(list(client.stream_intervals("/newton/prep")), [[-2, -1]])
-        client.stream_remove("/newton/prep")
-        eq_(list(client.stream_intervals("/newton/prep")), [])
-
-        # Intervals that end at zero shouldn't be any different
-        client.http.put("stream/insert", "", { "path": "/newton/prep",
-                                               "start": -1, "end": 0 })
-        eq_(list(client.stream_intervals("/newton/prep")), [[-1, 0]])
-        client.stream_remove("/newton/prep")
-        eq_(list(client.stream_intervals("/newton/prep")), [])
-
-        # Try forcing a server request with equal start and end
-        with assert_raises(ClientError) as e:
-            client.http.put("stream/insert", "", { "path": "/newton/prep",
-                                                   "start": 0, "end": 0 })
-        in_("400 Bad Request", str(e.exception))
-        in_("start must precede end", str(e.exception))
-
-        # Specify start/end (starts too late)
-        data = timestamper.TimestamperRate(testfile, start, 120)
-        with assert_raises(ClientError) as e:
-            result = client.stream_insert("/newton/prep", data,
-                                          start + 5, start + 120)
-        in_("400 Bad Request", str(e.exception))
-        in_("Data timestamp 1332511200.0 < start time 1332511205.0",
-            str(e.exception))
-
-        # Specify start/end (ends too early)
-        data = timestamper.TimestamperRate(testfile, start, 120)
-        with assert_raises(ClientError) as e:
-            result = client.stream_insert("/newton/prep", data,
-                                          start, start + 1)
-        in_("400 Bad Request", str(e.exception))
-        # Client chunks the input, so the exact timestamp here might change
-        # if the chunk positions change.
-        in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
-            str(e.exception))
-
-        # Now do the real load
-        data = timestamper.TimestamperRate(testfile, start, 120)
-        result = client.stream_insert("/newton/prep", data,
-                                      start, start + 119.999777)
-
-        # Verify the intervals.  Should be just one, even if the data
-        # was inserted in chunks, due to nilmdb interval concatenation.
-        intervals = list(client.stream_intervals("/newton/prep"))
-        eq_(intervals, [[start, start + 119.999777]])
-
-        # Try some overlapping data -- just insert it again
-        data = timestamper.TimestamperRate(testfile, start, 120)
-        with assert_raises(ClientError) as e:
-            result = client.stream_insert("/newton/prep", data)
-        in_("400 Bad Request", str(e.exception))
-        in_("verlap", str(e.exception))
-
-        client.close()
-
-    def test_client_05_extractremove(self):
-        # Misc tests for extract and remove.  Most of them are in test_cmdline.
-        client = nilmdb.Client(url = testurl)
-
-        for x in client.stream_extract("/newton/prep", 999123, 999124):
-            raise AssertionError("shouldn't be any data for this request")
-
-        with assert_raises(ClientError) as e:
-            client.stream_remove("/newton/prep", 123, 120)
-
-        # Test the exception we get if we nest requests
-        with assert_raises(Exception) as e:
-            for data in client.stream_extract("/newton/prep"):
-                x = client.stream_intervals("/newton/prep")
-        in_("nesting calls is not supported", str(e.exception))
-
-        # Test count
-        eq_(client.stream_count("/newton/prep"), 14400)
-
-        client.close()
-
-    def test_client_06_generators(self):
-        # A lot of the client functionality is already tested by test_cmdline,
-        # but this gets a bit more coverage that cmdline misses.
-        client = nilmdb.Client(url = testurl)
-
-        # Trigger a client error in generator
-        start = datetime_tz.datetime_tz.smartparse("20120323T2000")
-        end = datetime_tz.datetime_tz.smartparse("20120323T1000")
-        for function in [ client.stream_intervals, client.stream_extract ]:
-            with assert_raises(ClientError) as e:
-                function("/newton/prep",
-                         start.totimestamp(),
-                         end.totimestamp()).next()
-            in_("400 Bad Request", str(e.exception))
-            in_("start must precede end", str(e.exception))
-
-        # Trigger a curl error in generator
-        with assert_raises(ServerError) as e:
-            client.http.get_gen("http://nosuchurl/").next()
-
-        # Trigger a curl error in generator
-        with assert_raises(ServerError) as e:
-            client.http.get_gen("http://nosuchurl/").next()
-
-        # Check non-json version of string output
-        eq_(json.loads(client.http.get("/stream/list",retjson=False)),
-            client.http.get("/stream/list",retjson=True))
-
-        # Check non-json version of generator output
-        for (a, b) in itertools.izip(
-            client.http.get_gen("/stream/list",retjson=False),
-            client.http.get_gen("/stream/list",retjson=True)):
-            eq_(json.loads(a), b)
-
-        # Check PUT with generator out
-        with assert_raises(ClientError) as e:
-            client.http.put_gen("stream/insert", "",
-                                { "path": "/newton/prep",
-                                  "start": 0, "end": 0 }).next()
-        in_("400 Bad Request", str(e.exception))
-        in_("start must precede end", str(e.exception))
-
-        # Check 404 for missing streams
-        for function in [ client.stream_intervals, client.stream_extract ]:
-            with assert_raises(ClientError) as e:
-                function("/no/such/stream").next()
-            in_("404 Not Found", str(e.exception))
-            in_("No such stream", str(e.exception))
-
-        client.close()
-
-    def test_client_07_headers(self):
-        # Make sure that /stream/intervals and /stream/extract
-        # properly return streaming, chunked, text/plain response.
-        # Pokes around in client.http internals a bit to look at the
-        # response headers.
-
-        client = nilmdb.Client(url = testurl)
-        http = client.http
-
-        # Use a warning rather than returning a test failure, so that we can
-        # still disable chunked responses for debugging.
-
-        # Intervals
-        x = http.get("stream/intervals", { "path": "/newton/prep" },
-                            retjson=False)
-        lines_(x, 1)
-        if "Transfer-Encoding: chunked" not in http._headers:
-            warnings.warn("Non-chunked HTTP response for /stream/intervals")
-        if "Content-Type: text/plain;charset=utf-8" not in http._headers:
-            raise AssertionError("/stream/intervals is not text/plain:\n" +
-                                 http._headers)
-
-        # Extract
-        x = http.get("stream/extract",
-                            { "path": "/newton/prep",
-                              "start": "123",
-                              "end": "124" }, retjson=False)
-        if "Transfer-Encoding: chunked" not in http._headers:
-            warnings.warn("Non-chunked HTTP response for /stream/extract")
-        if "Content-Type: text/plain;charset=utf-8" not in http._headers:
-            raise AssertionError("/stream/extract is not text/plain:\n" +
-                                 http._headers)
-
-        # Make sure Access-Control-Allow-Origin gets set
-        if "Access-Control-Allow-Origin: " not in http._headers:
-            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
-                                 "header in /stream/extract response:\n" +
-                                 http._headers)
-
-        client.close()
-
-    def test_client_08_unicode(self):
-        # Basic Unicode tests
-        client = nilmdb.Client(url = testurl)
-
-        # Delete streams that exist
-        for stream in client.stream_list():
-            client.stream_destroy(stream[0])
-
-        # Database is empty
-        eq_(client.stream_list(), [])
-
-        # Create Unicode stream, match it
-        raw = [ u"/düsseldorf/raw", u"uint16_6" ]
-        prep = [ u"/düsseldorf/prep", u"uint16_6" ]
-        client.stream_create(*raw)
-        eq_(client.stream_list(), [raw])
-        eq_(client.stream_list(layout=raw[1]), [raw])
-        eq_(client.stream_list(path=raw[0]), [raw])
-        client.stream_create(*prep)
-        eq_(client.stream_list(), [prep, raw])
-
-        # Set / get metadata with Unicode keys and values
-        eq_(client.stream_get_metadata(raw[0]), {})
-        eq_(client.stream_get_metadata(prep[0]), {})
-        meta1 = { u"alpha": u"α",
-                  u"β": u"beta" }
-        meta2 = { u"alpha": u"α" }
-        meta3 = { u"β": u"beta" }
-        client.stream_set_metadata(prep[0], meta1)
-        client.stream_update_metadata(prep[0], {})
-        client.stream_update_metadata(raw[0], meta2)
-        client.stream_update_metadata(raw[0], meta3)
-        eq_(client.stream_get_metadata(prep[0]), meta1)
-        eq_(client.stream_get_metadata(raw[0]), meta1)
-        eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
-        eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
-
-        client.close()
-
-    def test_client_09_closing(self):
-        # Make sure we actually close sockets correctly.  New
-        # connections will block for a while if they're not, since the
-        # server will stop accepting new connections.
-        for test in [1, 2]:
-            start = time.time()
-            for i in range(50):
-                if time.time() - start > 15:
-                    raise AssertionError("Connections seem to be blocking... "
-                                         "probably not closing properly.")
-                if test == 1:
-                    # explicit close
-                    client = nilmdb.Client(url = testurl)
-                    with assert_raises(ClientError) as e:
-                        client.stream_remove("/newton/prep", 123, 120)
-                    client.close() # remove this to see the failure
-                elif test == 2:
-                    # use the context manager
-                    with nilmdb.Client(url = testurl) as c:
-                        with assert_raises(ClientError) as e:
-                            c.stream_remove("/newton/prep", 123, 120)
-
-    def test_client_10_context(self):
-        # Test using the client's stream insertion context manager to
-        # insert data.
-        client = nilmdb.Client(testurl)
-
-        client.stream_create("/context/test", "uint16_1")
-        with client.stream_insert_context("/context/test") as ctx:
-            # override _max_data to trigger frequent server updates
-            ctx._max_data = 15
-
-            with assert_raises(ValueError):
-                ctx.insert_line("100 1")
-
-            ctx.insert_line("100 1\n")
-            ctx.insert_iter([ "101 1\n",
-                              "102 1\n",
-                              "103 1\n" ])
-            ctx.insert_line("104 1\n")
-            ctx.insert_line("105 1\n")
-            ctx.finalize()
-
-            ctx.insert_line("106 1\n")
-            ctx.update_end(106.5)
-            ctx.finalize()
-            ctx.update_start(106.8)
-            ctx.insert_line("107 1\n")
-            ctx.insert_line("108 1\n")
-            ctx.insert_line("109 1\n")
-            ctx.insert_line("110 1\n")
-            ctx.insert_line("111 1\n")
-            ctx.update_end(113)
-            ctx.insert_line("112 1\n")
-            ctx.update_end(114)
-            ctx.insert_line("113 1\n")
-            ctx.update_end(115)
-            ctx.insert_line("114 1\n")
-            ctx.finalize()
-
-        with assert_raises(ClientError):
-            with client.stream_insert_context("/context/test", 100, 200) as ctx:
-                ctx.insert_line("115 1\n")
-
-        with assert_raises(ClientError):
-            with client.stream_insert_context("/context/test", 200, 300) as ctx:
-                ctx.insert_line("115 1\n")
-
-        with client.stream_insert_context("/context/test", 200, 300) as ctx:
-            # make sure our override wasn't permanent
-            ne_(ctx._max_data, 15)
-            ctx.insert_line("225 1\n")
-            ctx.finalize()
-
-        eq_(list(client.stream_intervals("/context/test")),
-            [ [ 100, 105.000001 ],
-              [ 106, 106.5 ],
-              [ 106.8, 115 ],
-              [ 200, 300 ] ])
-
-        client.stream_destroy("/context/test")
-        client.close()
-
-    def test_client_11_emptyintervals(self):
-        # Empty intervals are ok!  If recording detection events
-        # by inserting rows into the database, we want to be able to
-        # have an interval where no events occurred.  Test them here.
-        client = nilmdb.Client(testurl)
-        client.stream_create("/empty/test", "uint16_1")
-
-        def info():
-            result = []
-            for interval in list(client.stream_intervals("/empty/test")):
-                result.append((client.stream_count("/empty/test", *interval),
-                               interval))
-            return result
-
-        eq_(info(), [])
-
-        # Insert a region with just a few points
-        with client.stream_insert_context("/empty/test") as ctx:
-            ctx.update_start(100)
-            ctx.insert_line("140 1\n")
-            ctx.insert_line("150 1\n")
-            ctx.insert_line("160 1\n")
-            ctx.update_end(200)
-            ctx.finalize()
-
-        eq_(info(), [(3, [100, 200])])
-
-        # Delete chunk, which will leave one data point and two intervals
-        client.stream_remove("/empty/test", 145, 175)
-        eq_(info(), [(1, [100, 145]),
-                     (0, [175, 200])])
-
-        # Try also creating a completely empty interval from scratch,
-        # in a few different ways.
-        client.stream_insert_block("/empty/test", "", 300, 350)
-        client.stream_insert("/empty/test", [], 400, 450)
-        with client.stream_insert_context("/empty/test", 500, 550):
-            pass
-
-        # If enough timestamps aren't provided, empty streams won't be created.
-        client.stream_insert("/empty/test", [])
-        with client.stream_insert_context("/empty/test"):
-            pass
-        client.stream_insert("/empty/test", [], start = 600)
-        with client.stream_insert_context("/empty/test", start = 700):
-            pass
-        client.stream_insert("/empty/test", [], end = 850)
-        with client.stream_insert_context("/empty/test", end = 950):
-            pass
-
-        # Try various things that might cause problems
-        with client.stream_insert_context("/empty/test", 1000, 1050):
-            ctx.finalize() # inserts [1000, 1050]
-            ctx.finalize() # nothing
-            ctx.finalize() # nothing
-            ctx.insert_line("1100 1\n")
-            ctx.finalize() # inserts [1100, 1100.000001]
-            ctx.update_start(1199)
-            ctx.insert_line("1200 1\n")
-            ctx.update_end(1250)
-            ctx.finalize() # inserts [1199, 1250]
-            ctx.update_start(1299)
-            ctx.finalize() # nothing
-            ctx.update_end(1350)
-            ctx.finalize() # nothing
-            ctx.update_start(1400)
-            ctx.update_end(1450)
-            ctx.finalize()
-            # implicit last finalize inserts [1400, 1450]
-
-        # Check everything
-        eq_(info(), [(1, [100, 145]),
-                     (0, [175, 200]),
-                     (0, [300, 350]),
-                     (0, [400, 450]),
-                     (0, [500, 550]),
-                     (0, [1000, 1050]),
-                     (1, [1100, 1100.000001]),
-                     (1, [1199, 1250]),
-                     (0, [1400, 1450]),
-                     ])
-
-        # Clean up
-        client.stream_destroy("/empty/test")
-        client.close()

tests/test_cmdline.py

@@ -1,846 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import nilmdb
-from nilmdb.utils.printf import *
-import nilmdb.cmdline
-from nilmdb.utils import datetime_tz
-
-import unittest
-from nose.tools import *
-from nose.tools import assert_raises
-import itertools
-import os
-import re
-import shutil
-import sys
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
-import StringIO
-import shlex
-
-from testutil.helpers import *
-
-testdb = "tests/cmdline-testdb"
-
-def server_start(max_results = None, bulkdata_args = {}):
-    global test_server, test_db
-    # Start web app on a custom port
-    test_db = nilmdb.NilmDB(testdb, sync = False,
-                            max_results = max_results,
-                            bulkdata_args = bulkdata_args)
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
-                                fast_shutdown = True,
-                                force_traceback = False)
-    test_server.start(blocking = False)
-
-def server_stop():
-    global test_server, test_db
-    # Close web app
-    test_server.stop()
-    test_db.close()
-
-def setup_module():
-    global test_server, test_db
-    # Clear out DB
-    recursive_unlink(testdb)
-    server_start()
-
-def teardown_module():
-    server_stop()
-
-# Add an encoding property to StringIO so Python will convert Unicode
-# properly when writing or reading.
-class UTF8StringIO(StringIO.StringIO):
-    encoding = 'utf-8'
-
-class TestCmdline(object):
-
-    def run(self, arg_string, infile=None, outfile=None):
-        """Run a cmdline client with the specified argument string,
-        passing the given input.  Returns a tuple with the output and
-        exit code"""
-        # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
-        class stdio_wrapper:
-            def __init__(self, stdin, stdout, stderr):
-                self.io = (stdin, stdout, stderr)
-            def __enter__(self):
-                self.saved = ( sys.stdin, sys.stdout, sys.stderr )
-                ( sys.stdin, sys.stdout, sys.stderr ) = self.io
-            def __exit__(self, type, value, traceback):
-                ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
-        # Empty input if none provided
-        if infile is None:
-            infile = UTF8StringIO("")
-        # Capture stderr
-        errfile = UTF8StringIO()
-        if outfile is None:
-            # If no output file, capture stdout with stderr
-            outfile = errfile
-        with stdio_wrapper(infile, outfile, errfile) as s:
-            try:
-                # shlex doesn't support Unicode very well.  Encode the
-                # string as UTF-8 explicitly before splitting.
-                args = shlex.split(arg_string.encode('utf-8'))
-                nilmdb.cmdline.Cmdline(args).run()
-                sys.exit(0)
-            except SystemExit as e:
-                exitcode = e.code
-        captured = outfile.getvalue()
-        self.captured = captured
-        self.exitcode = exitcode
-
-    def ok(self, arg_string, infile = None):
-        self.run(arg_string, infile)
-        if self.exitcode != 0:
-            self.dump()
-            eq_(self.exitcode, 0)
-
-    def fail(self, arg_string, infile = None,
-             exitcode = None, require_error = True):
-        self.run(arg_string, infile)
-        if exitcode is not None and self.exitcode != exitcode:
-            # Wrong exit code
-            self.dump()
-            eq_(self.exitcode, exitcode)
-        if self.exitcode == 0:
-            # Success, when we wanted failure
-            self.dump()
-            ne_(self.exitcode, 0)
-        # Make sure the output contains the word "error" at the
-        # beginning of a line, but only if an exitcode wasn't
-        # specified.
-        if require_error and not re.search("^error",
-                                           self.captured, re.MULTILINE):
-            raise AssertionError("command failed, but output doesn't "
-                                 "contain the string 'error'")
-
-    def contain(self, checkstring):
-        in_(checkstring, self.captured)
-
-    def match(self, checkstring):
-        eq_(checkstring, self.captured)
-
-    def matchfile(self, file):
-        # Captured data should match file contents exactly
-        with open(file) as f:
-            contents = f.read()
-            if contents != self.captured:
-                print contents[1:1000] + "\n"
-                print self.captured[1:1000] + "\n"
-                raise AssertionError("captured data doesn't match " + file)
-
-    def matchfilecount(self, file):
-        # Last line of captured data should match the number of
-        # non-commented lines in file
-        count = 0
-        with open(file) as f:
-            for line in f:
-                if line[0] != '#':
-                    count += 1
-        eq_(self.captured.splitlines()[-1], sprintf("%d", count))
-
-    def dump(self):
-        printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
-
-    def test_01_basic(self):
-
-        # help
-        self.ok("--help")
-        self.contain("usage:")
-
-        # fail for no args
-        self.fail("")
-
-        # fail for no such option
-        self.fail("--nosuchoption")
-
-        # fail for bad command
-        self.fail("badcommand")
-
-        # try some URL constructions
-        self.fail("--url http://nosuchurl/ info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
-
-        self.fail("--url nosuchurl info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
-
-        self.fail("-u nosuchurl/foo info")
-        self.contain("Couldn't resolve host 'nosuchurl'")
-
-        self.fail("-u localhost:0 info")
-        self.contain("couldn't connect to host")
-
-        self.ok("-u localhost:12380 info")
-        self.ok("info")
-
-        # Duplicated arguments should fail, but this isn't implemented
-        # due to it being kind of a pain with argparse.
-        if 0:
-            self.fail("-u url1 -u url2 info")
-            self.contain("duplicated argument")
-
-            self.fail("list --detail --detail")
-            self.contain("duplicated argument")
-
-            self.fail("list --detail --path path1 --path path2")
-            self.contain("duplicated argument")
-
-            self.fail("extract --start 2000-01-01 --start 2001-01-02")
-            self.contain("duplicated argument")
-
-    def test_02_info(self):
-        self.ok("info")
-        self.contain("Server URL: http://localhost:12380/")
-        self.contain("Client version: " + nilmdb.__version__)
-        self.contain("Server version: " + test_server.version)
-        self.contain("Server database path")
-        self.contain("Server database size")
-        self.contain("Server database free space")
-
-    def test_03_createlist(self):
-        # Basic stream tests, like those in test_client.
-
-        # No streams
-        self.ok("list")
-        self.match("")
-
-        # Bad paths
-        self.fail("create foo/bar/baz PrepData")
-        self.contain("paths must start with /")
-
-        self.fail("create /foo PrepData")
-        self.contain("invalid path")
-
-        # Bad layout type
-        self.fail("create /newton/prep NoSuchLayout")
-        self.contain("no such layout")
-        self.fail("create /newton/prep float32_0")
-        self.contain("no such layout")
-        self.fail("create /newton/prep float33_1")
-        self.contain("no such layout")
-
-        # Create a few streams
-        self.ok("create /newton/zzz/rawnotch RawNotchedData")
-        self.ok("create /newton/prep PrepData")
-        self.ok("create /newton/raw RawData")
-
-        # Should not be able to create a stream with another stream as
-        # its parent
-        self.fail("create /newton/prep/blah PrepData")
-        self.contain("path is subdir of existing node")
-
-        # Should not be able to create a stream at a location that
-        # has other nodes as children
-        self.fail("create /newton/zzz PrepData")
-        self.contain("subdirs of this path already exist")
-
-        # Verify we got those 3 streams and they're returned in
-        # alphabetical order.
-        self.ok("list")
-        self.match("/newton/prep PrepData\n"
-                   "/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
-
-        # Match just one type or one path.  Also check
-        # that --path is optional
-        self.ok("list --path /newton/raw")
-        self.match("/newton/raw RawData\n")
-
-        self.ok("list /newton/raw")
-        self.match("/newton/raw RawData\n")
-
-        self.fail("list -p /newton/raw /newton/raw")
-        self.contain("too many paths")
-
-        self.ok("list --layout RawData")
-        self.match("/newton/raw RawData\n")
-
-        # Wildcard matches
-        self.ok("list --layout Raw*")
-        self.match("/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
-
-        self.ok("list --path *zzz* --layout Raw*")
-        self.match("/newton/zzz/rawnotch RawNotchedData\n")
-
-        self.ok("list *zzz* --layout Raw*")
-        self.match("/newton/zzz/rawnotch RawNotchedData\n")
-
-        self.ok("list --path *zzz* --layout Prep*")
-        self.match("")
-
-        # reversed range
-        self.fail("list /newton/prep --start 2020-01-01 --end 2000-01-01")
-        self.contain("start must precede end")
-
-    def test_04_metadata(self):
-        # Set / get metadata
-        self.fail("metadata")
-        self.fail("metadata --get")
-
-        self.ok("metadata /newton/prep")
-        self.match("")
-
-        self.ok("metadata /newton/raw --get")
-        self.match("")
-
-        self.ok("metadata /newton/prep --set "
-                "'description=The Data' "
-                "v_scale=1.234")
-        self.ok("metadata /newton/raw --update "
-                "'description=The Data'")
-        self.ok("metadata /newton/raw --update "
-                "v_scale=1.234")
-
-        # various parsing tests
-        self.ok("metadata /newton/raw --update foo=")
-        self.fail("metadata /newton/raw --update =bar")
-        self.fail("metadata /newton/raw --update foo==bar")
-        self.fail("metadata /newton/raw --update foo;bar")
-
-        # errors
-        self.fail("metadata /newton/nosuchstream foo=bar")
-        self.contain("unrecognized arguments")
-        self.fail("metadata /newton/nosuchstream")
-        self.contain("No stream at path")
-        self.fail("metadata /newton/nosuchstream --set foo=bar")
-        self.contain("No stream at path")
-
-        self.ok("metadata /newton/prep")
-        self.match("description=The Data\nv_scale=1.234\n")
-
-        self.ok("metadata /newton/prep --get")
-        self.match("description=The Data\nv_scale=1.234\n")
-
-        self.ok("metadata /newton/prep --get descr")
-        self.match("descr=\n")
-
-        self.ok("metadata /newton/prep --get description")
-        self.match("description=The Data\n")
-
-        self.ok("metadata /newton/prep --get description v_scale")
-        self.match("description=The Data\nv_scale=1.234\n")
-
-        self.ok("metadata /newton/prep --set "
-                "'description=The Data'")
-
-        self.ok("metadata /newton/prep --get")
-        self.match("description=The Data\n")
-
-        self.fail("metadata /newton/nosuchpath")
-        self.contain("No stream at path /newton/nosuchpath")
-
-    def test_05_parsetime(self):
-        os.environ['TZ'] = "America/New_York"
-        cmd = nilmdb.cmdline.Cmdline(None)
-        test = datetime_tz.datetime_tz.now()
-        eq_(cmd.parse_time(str(test)), test)
-        test = datetime_tz.datetime_tz.smartparse("20120405 1400-0400")
-        eq_(cmd.parse_time("hi there 20120405 1400-0400 testing! 123"), test)
-        eq_(cmd.parse_time("20120405 1800 UTC"), test)
-        eq_(cmd.parse_time("20120405 1400-0400 UTC"), test)
-        for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
-            with assert_raises(ValueError):
-                x = cmd.parse_time(badtime)
-        x = cmd.parse_time("now")
-        eq_(cmd.parse_time("snapshot-20120405-140000.raw.gz"), test)
-        eq_(cmd.parse_time("prep-20120405T1400"), test)
-
-    def test_06_insert(self):
-        self.ok("insert --help")
-
-        self.fail("insert /foo/bar baz qwer")
-        self.contain("error getting stream info")
-
-        self.fail("insert /newton/prep baz qwer")
-        self.match("error opening input file baz\n")
-
-        self.fail("insert /newton/prep")
-        self.contain("error extracting time")
-
-        self.fail("insert --start 19801205 /newton/prep 1 2 3 4")
-        self.contain("--start can only be used with one input file")
-
-        self.fail("insert /newton/prep "
-                  "tests/data/prep-20120323T1000")
-
-        # insert pre-timestamped data, from stdin
-        os.environ['TZ'] = "UTC"
-        with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
-
-        # insert pre-timestamped data, with bad times (non-monotonic)
-        os.environ['TZ'] = "UTC"
-        with open("tests/data/prep-20120323T1004-badtimes") as input:
-            self.fail("insert --none /newton/prep", input)
-            self.contain("error parsing input data")
-            self.contain("line 7:")
-            self.contain("timestamp is not monotonically increasing")
-
-        # insert data with normal timestamper from filename
-        os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1000 "
-                "tests/data/prep-20120323T1002")
-
-        # overlap
-        os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/prep "
-                  "tests/data/prep-20120323T1004")
-        self.contain("overlap")
-
-        # Just to help test more situations -- stop and restart
-        # the server now.  This tests nilmdb's interval caching,
-        # at the very least.
-        server_stop()
-        server_start()
-
-        # still an overlap if we specify a different start
-        os.environ['TZ'] = "America/New_York"
-        self.fail("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep"
-                  " tests/data/prep-20120323T1004")
-        self.contain("overlap")
-
-        # wrong format
-        os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/raw "
-                  "tests/data/prep-20120323T1004")
-        self.contain("error parsing input data")
-
-        # empty data does nothing
-        self.ok("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep "
-                "/dev/null")
-
-        # bad start time
-        self.fail("insert --rate 120 --start 'whatever' /newton/prep /dev/null")
-
-    def test_07_detail(self):
-        # Just count the number of lines, it's probably fine
-        self.ok("list --detail")
-        lines_(self.captured, 8)
-
-        self.ok("list --detail --path *prep")
-        lines_(self.captured, 4)
-
-        self.ok("list --detail --path *prep --start='23 Mar 2012 10:02'")
-        lines_(self.captured, 3)
-
-        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05'")
-        lines_(self.captured, 2)
-
-        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15'")
-        lines_(self.captured, 2)
-        self.contain("10:05:15.000")
-
-        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'")
-        lines_(self.captured, 2)
-        self.contain("10:05:15.500")
-
-        self.ok("list --detail --path *prep --start='23 Mar 2012 19:05:15.50'")
-        lines_(self.captured, 2)
-        self.contain("no intervals")
-
-        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'"
-                + " --end='23 Mar 2012 10:05:15.51'")
-        lines_(self.captured, 2)
-        self.contain("10:05:15.500")
-
-        self.ok("list --detail")
-        lines_(self.captured, 8)
-
-        # Verify the "raw timestamp" output
-        self.ok("list --detail --path *prep --timestamp-raw "
-                "--start='23 Mar 2012 10:05:15.50'")
-        lines_(self.captured, 2)
-        self.contain("[ 1332497115.5 -> 1332497159.991668 ]")
-
-        self.ok("list --detail --path *prep -T "
-                "--start='23 Mar 2012 10:05:15.612'")
-        lines_(self.captured, 2)
-        self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
-
-    def test_08_extract(self):
-        # nonexistent stream
-        self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")
-        self.contain("error getting stream info")
-
-        # reversed range
-        self.fail("extract -a /newton/prep --start 2020-01-01 --end 2000-01-01")
-        self.contain("start is after end")
-
-        # empty ranges return error 2
-        self.fail("extract -a /newton/prep " +
-                  "--start '23 Mar 2012 20:00:30' " +
-                  "--end '23 Mar 2012 20:00:31'",
-                  exitcode = 2, require_error = False)
-        self.contain("no data")
-        self.fail("extract -a /newton/prep " +
-                  "--start '23 Mar 2012 20:00:30.000001' " +
-                  "--end '23 Mar 2012 20:00:30.000002'",
-                  exitcode = 2, require_error = False)
-        self.contain("no data")
-        self.fail("extract -a /newton/prep " +
-                  "--start '23 Mar 2022 10:00:30' " +
-                  "--end '23 Mar 2022 10:00:31'",
-                  exitcode = 2, require_error = False)
-        self.contain("no data")
-
-        # but are ok if we're just counting results
-        self.ok("extract --count /newton/prep " +
-                "--start '23 Mar 2012 20:00:30' " +
-                "--end '23 Mar 2012 20:00:31'")
-        self.match("0\n")
-        self.ok("extract -c /newton/prep " +
-                "--start '23 Mar 2012 20:00:30.000001' " +
-                "--end '23 Mar 2012 20:00:30.000002'")
-        self.match("0\n")
-
-        # Check various dumps against stored copies of how they should appear
-        def test(file, start, end, extra=""):
-            self.ok("extract " + extra + " /newton/prep " +
-                    "--start '23 Mar 2012 " + start + "' " +
-                    "--end '23 Mar 2012 " + end + "'")
-            self.matchfile("tests/data/extract-" + str(file))
-            self.ok("extract --count " + extra + " /newton/prep " +
-                    "--start '23 Mar 2012 " + start + "' " +
-                    "--end '23 Mar 2012 " + end + "'")
-            self.matchfilecount("tests/data/extract-" + str(file))
-        test(1, "10:00:30", "10:00:31", extra="-a")
-        test(1, "10:00:30.000000", "10:00:31", extra="-a")
-        test(2, "10:00:30.000001", "10:00:31")
-        test(2, "10:00:30.008333", "10:00:31")
-        test(3, "10:00:30.008333", "10:00:30.008334")
-        test(3, "10:00:30.008333", "10:00:30.016667")
-        test(4, "10:00:30.008333", "10:00:30.025")
-        test(5, "10:00:30", "10:00:31", extra="--annotate --bare")
-        test(6, "10:00:30", "10:00:31", extra="-b")
-        test(7, "10:00:30", "10:00:30.999", extra="-a -T")
-        test(7, "10:00:30", "10:00:30.999", extra="-a --timestamp-raw")
-
-        # all data put in by tests
-        self.ok("extract -a /newton/prep --start 2000-01-01 --end 2020-01-01")
-        lines_(self.captured, 43204)
-        self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
-        self.match("43200\n")
-
-    def test_09_truncated(self):
-        # Test truncated responses by overriding the nilmdb max_results
-        server_stop()
-        server_start(max_results = 2)
-        self.ok("list --detail")
-        lines_(self.captured, 8)
-        server_stop()
-        server_start()
-
-    def test_10_remove(self):
-        # Removing data
-
-        # Try nonexistent stream
-        self.fail("remove /no/such/foo --start 2000-01-01 --end 2020-01-01")
-        self.contain("No stream at path")
-
-        # empty or backward ranges return errors
-        self.fail("remove /newton/prep --start 2020-01-01 --end 2000-01-01")
-        self.contain("start must precede end")
-
-        self.fail("remove /newton/prep " +
-                  "--start '23 Mar 2012 10:00:30' " +
-                  "--end '23 Mar 2012 10:00:30'")
-        self.contain("start must precede end")
-        self.fail("remove /newton/prep " +
-                  "--start '23 Mar 2012 10:00:30.000001' " +
-                  "--end '23 Mar 2012 10:00:30.000001'")
-        self.contain("start must precede end")
-        self.fail("remove /newton/prep " +
-                  "--start '23 Mar 2022 10:00:30' " +
-                  "--end '23 Mar 2022 10:00:30'")
-        self.contain("start must precede end")
-
-        # Verbose
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2022 20:00:30' " +
-                "--end '23 Mar 2022 20:00:31'")
-        self.match("0\n")
-        self.ok("remove --count /newton/prep " +
-                "--start '23 Mar 2022 20:00:30' " +
-                "--end '23 Mar 2022 20:00:31'")
-        self.match("0\n")
-
-        # Make sure we have the data we expect
-        self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
-                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:04:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:05:59.991668 +0000 ]\n")
-
-        # Remove various chunks of prep data and make sure
-        # they're gone.
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:30' " +
-                "--end '23 Mar 2012 10:00:40'")
-        self.match("1200\n")
-
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:10' " +
-                "--end '23 Mar 2012 10:00:20'")
-        self.match("1200\n")
-
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:05' " +
-                "--end '23 Mar 2012 10:00:25'")
-        self.match("1200\n")
-
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:03:50' " +
-                "--end '23 Mar 2012 10:06:50'")
-        self.match("15600\n")
-
-        self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
-        self.match("24000\n")
-
-        # See the missing chunks in list output
-        self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
-                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:00:40.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
-
-        # Remove all data, verify it's missing
-        self.ok("remove /newton/prep --start 2000-01-01 --end 2020-01-01")
-        self.match("")  # no count requested this time
-        self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
-                   "  (no intervals)\n")
-
-        # Reinsert some data, to verify that no overlaps with deleted
-        # data are reported
-        os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1000 "
-                "tests/data/prep-20120323T1002")
-
-    def test_11_destroy(self):
-        # Delete records
-        self.ok("destroy --help")
-
-        self.fail("destroy")
-        self.contain("too few arguments")
-
-        self.fail("destroy /no/such/stream")
-        self.contain("No stream at path")
-
-        self.fail("destroy asdfasdf")
-        self.contain("No stream at path")
-
-        # From previous tests, we have:
-        self.ok("list")
-        self.match("/newton/prep PrepData\n"
-                   "/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
-
-        # Notice how they're not empty
-        self.ok("list --detail")
-        lines_(self.captured, 7)
-
-        # Delete some
-        self.ok("destroy /newton/prep")
-        self.ok("list")
-        self.match("/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
-
-        self.ok("destroy /newton/zzz/rawnotch")
-        self.ok("list")
-        self.match("/newton/raw RawData\n")
-
-        self.ok("destroy /newton/raw")
-        self.ok("create /newton/raw RawData")
-        self.ok("destroy /newton/raw")
-        self.ok("list")
-        self.match("")
-
-        # Re-create a previously deleted location, and some new ones
-        rebuild = [ "/newton/prep", "/newton/zzz",
-                    "/newton/raw", "/newton/asdf/qwer" ]
-        for path in rebuild:
-            # Create the path
-            self.ok("create " + path + " PrepData")
-            self.ok("list")
-            self.contain(path)
-            # Make sure it was created empty
-            self.ok("list --detail --path " + path)
-            self.contain("(no intervals)")
-
-    def test_12_unicode(self):
-        # Unicode paths.
-        self.ok("destroy /newton/asdf/qwer")
-        self.ok("destroy /newton/prep")
-        self.ok("destroy /newton/raw")
-        self.ok("destroy /newton/zzz")
-
-        self.ok(u"create /düsseldorf/raw uint16_6")
-        self.ok("list --detail")
-        self.contain(u"/düsseldorf/raw uint16_6")
-        self.contain("(no intervals)")
-
-        # Unicode metadata
-        self.ok(u"metadata /düsseldorf/raw --set α=beta 'γ=δ'")
-        self.ok(u"metadata /düsseldorf/raw --update 'α=β ε τ α'")
-        self.ok(u"metadata /düsseldorf/raw")
-        self.match(u"α=β ε τ α\nγ=δ\n")
-
-        self.ok(u"destroy /düsseldorf/raw")
-
-    def test_13_files(self):
-        # Test BulkData's ability to split into multiple files,
-        # by forcing the file size to be really small.
-        server_stop()
-        server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
-                                       "files_per_dir" : 3 })
-
-        # Fill data
-        self.ok("create /newton/prep float32_8")
-        os.environ['TZ'] = "UTC"
-        with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
-
-        # Extract it
-        self.ok("extract /newton/prep --start '2000-01-01' " +
-                "--end '2012-03-23 10:04:01'")
-        lines_(self.captured, 120)
-        self.ok("extract /newton/prep --start '2000-01-01' " +
-                "--end '2022-03-23 10:04:01'")
-        lines_(self.captured, 14400)
-
-        # Make sure there were lots of files generated in the database
-        # dir
-        nfiles = 0
-        for (dirpath, dirnames, filenames) in os.walk(testdb):
-            nfiles += len(filenames)
-        assert(nfiles > 500)
-
-        # Make sure we can restart the server with a different file
-        # size and have it still work
-        server_stop()
-        server_start()
-        self.ok("extract /newton/prep --start '2000-01-01' " +
-                "--end '2022-03-23 10:04:01'")
-        lines_(self.captured, 14400)
-
-        # Now recreate the data one more time and make sure there are
-        # fewer files.
-        self.ok("destroy /newton/prep")
-        self.fail("destroy /newton/prep") # already destroyed
-        self.ok("create /newton/prep float32_8")
-        os.environ['TZ'] = "UTC"
-        with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
-        nfiles = 0
-        for (dirpath, dirnames, filenames) in os.walk(testdb):
-            nfiles += len(filenames)
-        lt_(nfiles, 50)
-        self.ok("destroy /newton/prep") # destroy again
-
-    def test_14_remove_files(self):
-        # Test BulkData's ability to remove when data is split into
-        # multiple files.  Should be a fairly comprehensive test of
-        # remove functionality.
-        server_stop()
-        server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
-                                       "files_per_dir" : 3 })
-
-        # Insert data.  Just for fun, insert out of order
-        self.ok("create /newton/prep PrepData")
-        os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1002 "
-                "tests/data/prep-20120323T1000")
-
-        # Should take up about 2.8 MB here (including directory entries)
-        du_before = nilmdb.utils.diskusage.du(testdb)
-
-        # Make sure we have the data we expect
-        self.ok("list --detail")
-        self.match("/newton/prep PrepData\n" +
-                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n")
-
-        # Remove various chunks of prep data and make sure
-        # they're gone.
-        self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
-        self.match("28800\n")
-
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:30' " +
-                "--end '23 Mar 2012 10:03:30'")
-        self.match("21600\n")
-
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:10' " +
-                "--end '23 Mar 2012 10:00:20'")
-        self.match("1200\n")
-
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:00:05' " +
-                "--end '23 Mar 2012 10:00:25'")
-        self.match("1200\n")
-
-        self.ok("remove -c /newton/prep " +
-                "--start '23 Mar 2012 10:03:50' " +
-                "--end '23 Mar 2012 10:06:50'")
-        self.match("1200\n")
-
-        self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
-        self.match("3600\n")
-
-        # See the missing chunks in list output
-        self.ok("list --detail")
-        self.match("/newton/prep PrepData\n" +
-                   "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
-                   "  [ Fri, 23 Mar 2012 10:03:30.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
-
-        # We have 1/8 of the data that we had before, so the file size
-        # should have dropped below 1/4 of what it used to be
-        du_after = nilmdb.utils.diskusage.du(testdb)
-        lt_(du_after, (du_before / 4))
-
-        # Remove anything that came from the 10:02 data file
-        self.ok("remove /newton/prep " +
-                "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
-
-        # Re-insert 19 lines from that file, then remove them again.
-        # With the specific file_size above, this will cause the last
-        # file in the bulk data storage to be exactly file_size large,
-        # so removing the data should also remove that last file.
-        self.ok("insert --rate 120 /newton/prep " +
-                "tests/data/prep-20120323T1002-first19lines")
-        self.ok("remove /newton/prep " +
-                "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
-
-        # Shut down and restart server, to force nrows to get refreshed.
-        server_stop()
-        server_start()
-
-        # Re-add the full 10:02 data file.  This tests adding new data once
-        # we removed data near the end.
-        self.ok("insert --rate 120 /newton/prep tests/data/prep-20120323T1002")
-
-        # See if we can extract it all
-        self.ok("extract /newton/prep --start 2000-01-01 --end 2020-01-01")
-        lines_(self.captured, 15600)

tests/test_interval.py

@@ -1,373 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import nilmdb
-from nilmdb.utils.printf import *
-from nilmdb.utils import datetime_tz
-
-from nose.tools import *
-from nose.tools import assert_raises
-import itertools
-
-from nilmdb.server.interval import (Interval, DBInterval,
-                                    IntervalSet, IntervalError)
-
-from testutil.helpers import *
-import unittest
-
-# set to False to skip live renders
-do_live_renders = False
-def render(iset, description = "", live = True):
-    import testutil.renderdot as renderdot
-    r = renderdot.RBTreeRenderer(iset.tree)
-    return r.render(description, live and do_live_renders)
-
-def makeset(string):
-    """Build an IntervalSet from a string, for testing purposes
-
-    Each character is 1 second
-    [ = interval start
-    | = interval end + next start
-    ] = interval end
-    . = zero-width interval (identical start and end)
-    anything else is ignored
-    """
-    iset = IntervalSet()
-    for i, c in enumerate(string):
-        day = i + 10000
-        if (c == "["):
-            start = day
-        elif (c == "|"):
-            iset += Interval(start, day)
-            start = day
-        elif (c == ")"):
-            iset += Interval(start, day)
-            del start
-        elif (c == "."):
-            iset += Interval(day, day)
-    return iset
-
-class TestInterval:
-    def test_interval(self):
-        # Test Interval class
-        os.environ['TZ'] = "America/New_York"
-        datetime_tz._localtz = None
-        (d1, d2, d3) = [ datetime_tz.datetime_tz.smartparse(x).totimestamp()
-                         for x in [ "03/24/2012", "03/25/2012", "03/26/2012" ] ]
-
-        # basic construction
-        i = Interval(d1, d2)
-        i = Interval(d1, d3)
-        eq_(i.start, d1)
-        eq_(i.end, d3)
-
-        # assignment is allowed, but not verified
-        i.start = d2
-        #with assert_raises(IntervalError):
-        #    i.end = d1
-        i.start = d1
-        i.end = d2
-
-        # end before start
-        with assert_raises(IntervalError):
-            i = Interval(d3, d1)
-
-        # compare
-        assert(Interval(d1, d2) == Interval(d1, d2))
-        assert(Interval(d1, d2) < Interval(d1, d3))
-        assert(Interval(d1, d3) > Interval(d1, d2))
-        assert(Interval(d1, d2) < Interval(d2, d3))
-        assert(Interval(d1, d3) < Interval(d2, d3))
-        assert(Interval(d2, d2+0.01) > Interval(d1, d3))
-        assert(Interval(d3, d3+0.01) == Interval(d3, d3+0.01))
-        #with assert_raises(TypeError): # was AttributeError, that's wrong
-        #    x = (i == 123)
-
-        # subset
-        eq_(Interval(d1, d3).subset(d1, d2), Interval(d1, d2))
-        with assert_raises(IntervalError):
-            x = Interval(d2, d3).subset(d1, d2)
-
-        # big integers and floats
-        x = Interval(5000111222, 6000111222)
-        eq_(str(x), "[5000111222.0 -> 6000111222.0)")
-        x = Interval(123.45, 234.56)
-        eq_(str(x), "[123.45 -> 234.56)")
-
-        # misc
-        i = Interval(d1, d2)
-        eq_(repr(i), repr(eval(repr(i))))
-        eq_(str(i), "[1332561600.0 -> 1332648000.0)")
-
-    def test_interval_intersect(self):
-        # Test Interval intersections
-        dates = [ 100, 200, 300, 400 ]
-        perm = list(itertools.permutations(dates, 2))
-        prod = list(itertools.product(perm, perm))
-        should_intersect = {
-            False: [4, 5, 8, 20, 48, 56, 60, 96, 97, 100],
-            True: [0, 1, 2, 12, 13, 14, 16, 17, 24, 25, 26, 28, 29,
-                   32, 49, 50, 52, 53, 61, 62, 64, 65, 68, 98, 101, 104]
-            }
-        for i,((a,b),(c,d)) in enumerate(prod):
-            try:
-                i1 = Interval(a, b)
-                i2 = Interval(c, d)
-                eq_(i1.intersects(i2), i2.intersects(i1))
-                in_(i, should_intersect[i1.intersects(i2)])
-            except IntervalError:
-                assert(i not in should_intersect[True] and
-                       i not in should_intersect[False])
-        with assert_raises(TypeError):
-            x = i1.intersects(1234)
-
-    def test_intervalset_construct(self):
-        # Test IntervalSet construction
-        dates = [ 100, 200, 300, 400 ]
-
-        a = Interval(dates[0], dates[1])
-        b = Interval(dates[1], dates[2])
-        c = Interval(dates[0], dates[2])
-        d = Interval(dates[2], dates[3])
-
-        iseta = IntervalSet(a)
-        isetb = IntervalSet([a, b])
-        isetc = IntervalSet([a])
-        ne_(iseta, isetb)
-        eq_(iseta, isetc)
-        with assert_raises(TypeError):
-            x = iseta != 3
-        ne_(IntervalSet(a), IntervalSet(b))
-
-        # Note that assignment makes a new reference (not a copy)
-        isetd = IntervalSet(isetb)
-        isete = isetd
-        eq_(isetd, isetb)
-        eq_(isetd, isete)
-        isetd -= a
-        ne_(isetd, isetb)
-        eq_(isetd, isete)
-
-        # test iterator
-        for interval in iseta:
-            pass
-
-        # overlap
-        with assert_raises(IntervalError):
-            x = IntervalSet([a, b, c])
-
-        # bad types
-        with assert_raises(Exception):
-            x = IntervalSet([1, 2])
-
-        iset = IntervalSet(isetb)   # test iterator
-        eq_(iset, isetb)
-        eq_(len(iset), 2)
-        eq_(len(IntervalSet()), 0)
-
-        # Test adding
-        iset = IntervalSet(a)
-        iset += IntervalSet(b)
-        eq_(iset, IntervalSet([a, b]))
-
-        iset = IntervalSet(a)
-        iset += b
-        eq_(iset, IntervalSet([a, b]))
-
-        iset = IntervalSet(a)
-        iset.iadd_nocheck(b)
-        eq_(iset, IntervalSet([a, b]))
-
-        iset = IntervalSet(a) + IntervalSet(b)
-        eq_(iset, IntervalSet([a, b]))
-
-        iset = IntervalSet(b) + a
-        eq_(iset, IntervalSet([a, b]))
-
-        # A set consisting of [0-1],[1-2] should match a set consisting of [0-2]
-        eq_(IntervalSet([a,b]), IntervalSet([c]))
-        # Etc
-        ne_(IntervalSet([a,d]), IntervalSet([c]))
-        ne_(IntervalSet([c]), IntervalSet([a,d]))
-        ne_(IntervalSet([c,d]), IntervalSet([b,d]))
-
-        # misc
-        eq_(repr(iset), repr(eval(repr(iset))))
-        eq_(str(iset), "[[100.0 -> 200.0), [200.0 -> 300.0)]")
-
-    def test_intervalset_geniset(self):
-        # Test basic iset construction
-        eq_(makeset("  [----)   "),
-            makeset("  [-|--)   "))
-
-        eq_(makeset("[)  [--)   ") +
-            makeset(" [)    [--)"),
-            makeset("[|) [-----)"))
-
-        eq_(makeset("  [-------)"),
-            makeset("  [-|-----|"))
-
-
-    def test_intervalset_intersect(self):
-        # Test intersection (&)
-        with assert_raises(TypeError): # was AttributeError
-            x = makeset("[--)") & 1234
-
-        # Intersection with interval
-        eq_(makeset("[---|---)[)") &
-            list(makeset("  [------) "))[0],
-            makeset("  [-----)  "))
-
-        # Intersection with sets
-        eq_(makeset("[---------)") &
-            makeset(" [---)     "),
-            makeset(" [---)     "))
-
-        eq_(makeset(" [---)     ") &
-            makeset("[---------)"),
-            makeset(" [---)     "))
-
-        eq_(makeset("    [-----)") &
-            makeset(" [-----)   "),
-            makeset("    [--)   "))
-
-        eq_(makeset(" [--)  [--)") &
-            makeset("  [------) "),
-            makeset("  [-)  [-) "))
-
-        eq_(makeset("      [---)") &
-            makeset(" [--)      "),
-            makeset("           "))
-
-        eq_(makeset("    [-|---)") &
-            makeset(" [-----|-) "),
-            makeset("    [----) "))
-
-        eq_(makeset("    [-|-)  ") &
-            makeset(" [-|--|--) "),
-            makeset("    [---)  "))
-
-        # Border cases -- will give different results if intervals are
-        # half open or fully closed.  Right now, they are half open,
-        # although that's a little messy since the database intervals
-        # often contain a data point at the endpoint.
-        half_open = True
-        if half_open:
-            eq_(makeset("      [---)") &
-                makeset(" [----)    "),
-                makeset("           "))
-            eq_(makeset(" [----)[--)") &
-                makeset("[-) [--) [)"),
-                makeset(" [) [-)  [)"))
-        else:
-            eq_(makeset("      [---)") &
-                makeset(" [----)    "),
-                makeset("      .    "))
-            eq_(makeset(" [----)[--)") &
-                makeset("[-) [--) [)"),
-                makeset(" [) [-). [)"))
-
-class TestIntervalDB:
-    def test_dbinterval(self):
-        # Test DBInterval class
-        i = DBInterval(100, 200, 100, 200, 10000, 20000)
-        eq_(i.start, 100)
-        eq_(i.end, 200)
-        eq_(i.db_start, 100)
-        eq_(i.db_end, 200)
-        eq_(i.db_startpos, 10000)
-        eq_(i.db_endpos, 20000)
-        eq_(repr(i), repr(eval(repr(i))))
-
-        # end before start
-        with assert_raises(IntervalError):
-            i = DBInterval(200, 100, 100, 200, 10000, 20000)
-
-        # db_start too late
-        with assert_raises(IntervalError):
-            i = DBInterval(100, 200, 150, 200, 10000, 20000)
-
-        # db_end too soon
-        with assert_raises(IntervalError):
-            i = DBInterval(100, 200, 100, 150, 10000, 20000)
-
-        # actual start, end can be a subset
-        a = DBInterval(150, 200, 100, 200, 10000, 20000)
-        b = DBInterval(100, 150, 100, 200, 10000, 20000)
-        c = DBInterval(150, 160, 100, 200, 10000, 20000)
-
-        # Make a set of DBIntervals
-        iseta = IntervalSet([a, b])
-        isetc = IntervalSet(c)
-        assert(iseta.intersects(a))
-        assert(iseta.intersects(b))
-
-        # Test subset
-        with assert_raises(IntervalError):
-            x = a.subset(150, 250)
-
-        # Subset of those IntervalSets should still contain DBIntervals
-        for i in IntervalSet(iseta.intersection(Interval(125,250))):
-            assert(isinstance(i, DBInterval))
-
-class TestIntervalTree:
-
-    def test_interval_tree(self):
-        import random
-        random.seed(1234)
-
-        # make a set of 100 intervals
-        iset = IntervalSet()
-        j = 100
-        for i in random.sample(xrange(j),j):
-            interval = Interval(i, i+1)
-            iset += interval
-        render(iset, "Random Insertion")
-
-        # remove about half of them
-        for i in random.sample(xrange(j),j):
-            if random.randint(0,1):
-                iset -= Interval(i, i+1)
-
-        # try removing an interval that doesn't exist
-        with assert_raises(IntervalError):
-            iset -= Interval(1234,5678)
-        render(iset, "Random Insertion, deletion")
-
-        # make a set of 100 intervals, inserted in order
-        iset = IntervalSet()
-        j = 100
-        for i in xrange(j):
-            interval = Interval(i, i+1)
-            iset += interval
-        render(iset, "In-order insertion")
-
-class TestIntervalSpeed:
-    @unittest.skip("this is slow")
-    def test_interval_speed(self):
-        import yappi
-        import time
-        import testutil.aplotter as aplotter
-        import random
-        import math
-
-        print
-        yappi.start()
-        speeds = {}
-        limit = 10 # was 20
-        for j in [ 2**x for x in range(5,limit) ]:
-            start = time.time()
-            iset = IntervalSet()
-            for i in random.sample(xrange(j),j):
-                interval = Interval(i, i+1)
-                iset += interval
-            speed = (time.time() - start) * 1000000.0
-            printf("%d: %g μs (%g μs each, O(n log n) ratio %g)\n",
-                   j,
-                   speed,
-                   speed/j,
-                   speed / (j*math.log(j))) # should be constant
-            speeds[j] = speed
-        aplotter.plot(speeds.keys(), speeds.values(), plot_slope=True)
-        yappi.stop()
-        yappi.print_stats(sort_type=yappi.SORTTYPE_TTOT, limit=10)
-

tests/test_iteratorizer.py

@@ -1,61 +0,0 @@
-import nilmdb
-from nilmdb.utils.printf import *
-
-import nose
-from nose.tools import *
-from nose.tools import assert_raises
-import threading
-import time
-
-from testutil.helpers import *
-
-def func_with_callback(a, b, callback):
-    callback(a)
-    callback(b)
-    callback(a+b)
-    return "return value"
-
-class TestIteratorizer(object):
-    def test(self):
-
-        # First try it with a normal callback
-        self.result = ""
-        def cb(x):
-            self.result += str(x)
-        func_with_callback(1, 2, cb)
-        eq_(self.result, "123")
-
-        # Now make it an iterator
-        result = ""
-        f = lambda x: func_with_callback(1, 2, x)
-        with nilmdb.utils.Iteratorizer(f) as it:
-            for i in it:
-                result += str(i)
-        eq_(result, "123")
-        eq_(it.retval, "return value")
-
-        # Make sure things work when an exception occurs
-        result = ""
-        with nilmdb.utils.Iteratorizer(
-            lambda x: func_with_callback(1, "a", x)) as it:
-            with assert_raises(TypeError) as e:
-                for i in it:
-                    result += str(i)
-        eq_(result, "1a")
-
-        # Now try to trigger the case where we stop iterating
-        # mid-generator, and expect the iteratorizer to clean up after
-        # itself.  This doesn't have a particular result in the test,
-        # but gains coverage.
-        def foo():
-            with nilmdb.utils.Iteratorizer(f) as it:
-                it.next()
-        foo()
-        eq_(it.retval, None)
-
-        # Do the same thing when the curl hack is applied
-        def foo():
-            with nilmdb.utils.Iteratorizer(f, curl_hack = True) as it:
-                it.next()
-        foo()
-        eq_(it.retval, None)

tests/test_layout.py

@@ -1,266 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import nilmdb
-
-from nilmdb.utils.printf import *
-
-from nose.tools import *
-from nose.tools import assert_raises
-import distutils.version
-import itertools
-import os
-import shutil
-import sys
-import cherrypy
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
-import cStringIO
-import random
-import unittest
-
-from testutil.helpers import *
-
-from nilmdb.server.layout import *
-
-class TestLayouts(object):
-    # Some nilmdb.layout tests.  Not complete, just fills in missing
-    # coverage.
-    def test_layouts(self):
-        x = nilmdb.server.layout.get_named("PrepData")
-        y = nilmdb.server.layout.get_named("float32_8")
-        eq_(x.count, y.count)
-        eq_(x.datatype, y.datatype)
-        y = nilmdb.server.layout.get_named("float32_7")
-        ne_(x.count, y.count)
-        eq_(x.datatype, y.datatype)
-
-    def test_parsing(self):
-        self.real_t_parsing("PrepData", "RawData", "RawNotchedData")
-        self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
-    def real_t_parsing(self, name_prep, name_raw, name_rawnotch):
-        # invalid layouts
-        with assert_raises(TypeError) as e:
-            parser = Parser("NoSuchLayout")
-        with assert_raises(TypeError) as e:
-            parser = Parser("float32")
-
-        # too little data
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5\n")
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("error", str(e.exception))
-
-        # too much data
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n")
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("error", str(e.exception))
-
-        # just right
-        parser = Parser(name_prep)
-        data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n" +
-                 "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n")
-        parser.parse(data)
-        eq_(parser.min_timestamp, 1234567890.0)
-        eq_(parser.max_timestamp, 1234567890.1)
-        eq_(parser.data, [[1234567890.0,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8],
-                          [1234567890.1,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]])
-
-        # try RawData too, with clamping
-        parser = Parser(name_raw)
-        data = ( "1234567890.000000 1 2 3 4 5 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        parser.parse(data)
-        eq_(parser.data, [[1234567890.0,1,2,3,4,5,6],
-                          [1234567890.1,1,2,3,4,5,6]])
-
-        # pass an instantiated class
-        parser = Parser(get_named(name_rawnotch))
-        data = ( "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
-                 "1234567890.100000 1 2 3 4 5 6 7 8 9\n" )
-        parser.parse(data)
-
-        # non-monotonic
-        parser = Parser(name_raw)
-        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.099999 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("not monotonically increasing", str(e.exception))
-
-        parser = Parser(name_raw)
-        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("not monotonically increasing", str(e.exception))
-
-        parser = Parser(name_raw)
-        data = ( "1234567890.100000 1 2 3 4 5 6\n" +
-                 "1234567890.100001 1 2 3 4 5 6\n" )
-        parser.parse(data)
-
-        # RawData with values out of bounds
-        parser = Parser(name_raw)
-        data = ( "1234567890.000000 1 2 3 4 500000 6\n" +
-                 "1234567890.100000 1 2 3 4 5 6\n" )
-        with assert_raises(ParserError) as e:
-            parser.parse(data)
-        in_("value out of range", str(e.exception))
-
-        # Empty data should work but is useless
-        parser = Parser(name_raw)
-        data = ""
-        parser.parse(data)
-        assert(parser.min_timestamp is None)
-        assert(parser.max_timestamp is None)
-
-    def test_formatting(self):
-        self.real_t_formatting("PrepData", "RawData", "RawNotchedData")
-        self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
-    def real_t_formatting(self, name_prep, name_raw, name_rawnotch):
-        # invalid layout
-        with assert_raises(TypeError) as e:
-            formatter = Formatter("NoSuchLayout")
-
-        # too little data
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5 ],
-                 [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5 ] ]
-        with assert_raises(FormatterError) as e:
-            formatter.format(data)
-        in_("error", str(e.exception))
-
-        # too much data
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
-        with assert_raises(FormatterError) as e:
-            formatter.format(data)
-        in_("error", str(e.exception))
-
-        # just right
-        formatter = Formatter(name_prep)
-        data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ],
-                 [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1.100000 2.200000 3.300000 4.400000 " +
-            "5.500000 6.600000 7.700000 8.800000\n" +
-            "1234567890.100000 1.100000 2.200000 3.300000 4.400000 " +
-            "5.500000 6.600000 7.700000 8.800000\n")
-
-        # try RawData too
-        formatter = Formatter(name_raw)
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1 2 3 4 5 6\n" +
-            "1234567890.100000 1 2 3 4 5 6\n")
-
-        # pass an instantiated class
-        formatter = Formatter(get_named(name_rawnotch))
-        data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
-                 [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
-        text = formatter.format(data)
-        eq_(text,
-            "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
-            "1234567890.100000 1 2 3 4 5 6 7 8 9\n")
-
-        # Empty data should work but is useless
-        formatter = Formatter(name_raw)
-        data = []
-        text = formatter.format(data)
-        eq_(text, "")
-
-    def test_roundtrip(self):
-        self.real_t_roundtrip("PrepData", "RawData", "RawNotchedData")
-        self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
-    def real_t_roundtrip(self, name_prep, name_raw, name_rawnotch):
-        # Verify that textual data passed into the Parser, and then
-        # back through the Formatter, then back into the Parser,
-        # gives identical parsed representations
-        random.seed(12345)
-
-        def do_roundtrip(layout, datagen):
-            for i in range(100):
-                rows = random.randint(1,100)
-                data = ""
-                ts = 1234567890
-                for r in range(rows):
-                    ts += random.uniform(0,1)
-                    row = sprintf("%f", ts) + " "
-                    row += " ".join(datagen())
-                    row += "\n"
-                    data += row
-                parser1 = Parser(layout)
-                formatter = Formatter(layout)
-                parser2 = Parser(layout)
-                parser1.parse(data)
-                parser2.parse(formatter.format(parser1.data))
-                eq_(parser1.data, parser2.data)
-
-        def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
-                     for x in range(8) ]
-        do_roundtrip(name_prep, datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(6) ]
-        do_roundtrip(name_raw, datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(9) ]
-        do_roundtrip(name_rawnotch, datagen)
-
-class TestLayoutSpeed:
-    @unittest.skip("this is slow")
-    def test_layout_speed(self):
-        import time
-
-        random.seed(54321)
-
-        def do_speedtest(layout, datagen, rows = 5000, times = 100):
-            # Build data once
-            data = ""
-            ts = 1234567890
-            for r in range(rows):
-                ts += random.uniform(0,1)
-                row = sprintf("%f", ts) + " "
-                row += " ".join(datagen())
-                row += "\n"
-                data += row
-
-            # Do lots of roundtrips
-            start = time.time()
-            for i in range(times):
-                parser = Parser(layout)
-                formatter = Formatter(layout)
-                parser.parse(data)
-                data = formatter.format(parser.data)
-            elapsed = time.time() - start
-            printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
-                   layout,
-                   elapsed * 1e3,
-                   (elapsed * 1e6) / (rows * times),
-                   (rows * times) / elapsed)
-
-        print ""
-        def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
-                     for x in range(10) ]
-        do_speedtest("float32_10", datagen)
-
-        def datagen():
-            return [ sprintf("%d", random.randint(0,65535))
-                     for x in range(10) ]
-        do_speedtest("uint16_10", datagen)

tests/test_lrucache.py

@@ -1,83 +0,0 @@
-import nilmdb
-from nilmdb.utils.printf import *
-
-import nose
-from nose.tools import *
-from nose.tools import assert_raises
-import threading
-import time
-import inspect
-
-from testutil.helpers import *
-
-@nilmdb.utils.lru_cache(size = 3)
-def foo1(n):
-    return n
-
-@nilmdb.utils.lru_cache(size = 5)
-def foo2(n):
-    return n
-
-def foo3d(n):
-    foo3d.destructed.append(n)
-foo3d.destructed = []
-@nilmdb.utils.lru_cache(size = 3, onremove = foo3d)
-def foo3(n):
-    return n
-
-class Foo:
-    def __init__(self):
-        self.calls = 0
-    @nilmdb.utils.lru_cache(size = 3, keys = slice(1, 2))
-    def foo(self, n, **kwargs):
-        self.calls += 1
-
-class TestLRUCache(object):
-    def test(self):
-
-        [ foo1(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_(foo1.cache_info(), (6, 3))
-        [ foo1(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_(foo1.cache_info(), (15, 3))
-        [ foo1(n) for n in [ 4, 2, 1, 1, 4 ] ]
-        eq_(foo1.cache_info(), (18, 5))
-
-        [ foo2(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_(foo2.cache_info(), (6, 3))
-        [ foo2(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_(foo2.cache_info(), (15, 3))
-        [ foo2(n) for n in [ 4, 2, 1, 1, 4 ] ]
-        eq_(foo2.cache_info(), (19, 4))
-
-        [ foo3(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_(foo3.cache_info(), (6, 3))
-        [ foo3(n) for n in [ 1, 2, 3, 1, 2, 3, 1, 2, 3 ] ]
-        eq_(foo3.cache_info(), (15, 3))
-        [ foo3(n) for n in [ 4, 2, 1, 1, 4 ] ]
-        eq_(foo3.cache_info(), (18, 5))
-        eq_(foo3d.destructed, [1, 3])
-        with assert_raises(KeyError):
-            foo3.cache_remove(1,2,3)
-        foo3.cache_remove(1)
-        eq_(foo3d.destructed, [1, 3, 1])
-        foo3.cache_remove_all()
-        eq_(foo3d.destructed, [1, 3, 1, 2, 4 ])
-
-        foo = Foo()
-        foo.foo(5)
-        foo.foo(6)
-        foo.foo(7)
-        foo.foo(5)
-        eq_(foo.calls, 3)
-
-        # Can't handle keyword arguments right now
-        with assert_raises(NotImplementedError):
-            foo.foo(3, asdf = 7)
-
-        # Verify that argspecs were maintained
-        eq_(inspect.getargspec(foo1),
-            inspect.ArgSpec(args=['n'],
-                            varargs=None, keywords=None, defaults=None))
-        eq_(inspect.getargspec(foo.foo),
-            inspect.ArgSpec(args=['self', 'n'],
-                            varargs=None, keywords="kwargs", defaults=None))

tests/test_mustclose.py

@@ -1,110 +0,0 @@
-import nilmdb
-from nilmdb.utils.printf import *
-
-import nose
-from nose.tools import *
-from nose.tools import assert_raises
-
-from testutil.helpers import *
-
-import sys
-import cStringIO
-import gc
-
-import inspect
-
-err = cStringIO.StringIO()
-
-@nilmdb.utils.must_close(errorfile = err)
-class Foo:
-    def __init__(self, arg):
-        fprintf(err, "Init %s\n", arg)
-
-    def __del__(self):
-        fprintf(err, "Deleting\n")
-
-    def close(self):
-        fprintf(err, "Closing\n")
-
-@nilmdb.utils.must_close(errorfile = err, wrap_verify = True)
-class Bar:
-    def __init__(self):
-        fprintf(err, "Init\n")
-
-    def __del__(self):
-        fprintf(err, "Deleting\n")
-
-    def close(self):
-        fprintf(err, "Closing\n")
-
-    def blah(self, arg):
-        fprintf(err, "Blah %s\n", arg)
-
-@nilmdb.utils.must_close(errorfile = err)
-class Baz:
-    pass
-
-class TestMustClose(object):
-    def test(self):
-
-        # Note: this test might fail if the Python interpreter doesn't
-        # garbage collect the object (and call its __del__ function)
-        # right after a "del x".
-
-        # Trigger error
-        err.truncate()
-        x = Foo("hi")
-        # Verify that the arg spec was maintained
-        eq_(inspect.getargspec(x.__init__),
-            inspect.ArgSpec(args = ['self', 'arg'],
-                            varargs = None, keywords = None, defaults = None))
-        del x
-        gc.collect()
-        eq_(err.getvalue(),
-            "Init hi\n"
-            "error: Foo.close() wasn't called!\n"
-            "Deleting\n")
-
-        # No error
-        err.truncate(0)
-        y = Foo("bye")
-        y.close()
-        del y
-        gc.collect()
-        eq_(err.getvalue(),
-            "Init bye\n"
-            "Closing\n"
-            "Deleting\n")
-
-        # Verify function calls when wrap_verify is True
-        err.truncate(0)
-        z = Bar()
-        eq_(inspect.getargspec(z.blah),
-            inspect.ArgSpec(args = ['self', 'arg'],
-                            varargs = None, keywords = None, defaults = None))
-        z.blah("boo")
-        z.close()
-        with assert_raises(AssertionError) as e:
-            z.blah("hello")
-        in_("called <function blah at 0x", str(e.exception))
-        in_("> after close", str(e.exception))
-        # Since the most recent assertion references 'z',
-        # we need to raise another assertion here so that
-        # 'z' will get properly deleted.
-        with assert_raises(AssertionError):
-            raise AssertionError()
-        del z
-        gc.collect()
-        eq_(err.getvalue(),
-            "Init\n"
-            "Blah boo\n"
-            "Closing\n"
-            "Deleting\n")
-
-        # Class with missing methods
-        err.truncate(0)
-        w = Baz()
-        w.close()
-        del w
-        eq_(err.getvalue(), "")
-

tests/test_nilmdb.py

@@ -1,212 +0,0 @@
-import nilmdb
-
-from nose.tools import *
-from nose.tools import assert_raises
-import distutils.version
-import simplejson as json
-import itertools
-import os
-import shutil
-import sys
-import cherrypy
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
-import cStringIO
-import time
-
-testdb = "tests/testdb"
-
-#@atexit.register
-#def cleanup():
-#    os.unlink(testdb)
-
-from testutil.helpers import *
-
-class Test00Nilmdb(object):  # named 00 so it runs first
-    def test_NilmDB(self):
-        recursive_unlink(testdb)
-
-        with assert_raises(IOError):
-            nilmdb.NilmDB("/nonexistant-db/foo")
-
-        db = nilmdb.NilmDB(testdb)
-        db.close()
-        db = nilmdb.NilmDB(testdb, sync=False)
-        db.close()
-
-        # test timer, just to get coverage
-        capture = cStringIO.StringIO()
-        old = sys.stdout
-        sys.stdout = capture
-        with nilmdb.utils.Timer("test"):
-            time.sleep(0.01)
-        sys.stdout = old
-        in_("test: ", capture.getvalue())
-
-    def test_stream(self):
-        db = nilmdb.NilmDB(testdb, sync=False)
-        eq_(db.stream_list(), [])
-
-        # Bad path
-        with assert_raises(ValueError):
-            db.stream_create("foo/bar/baz", "PrepData")
-        with assert_raises(ValueError):
-            db.stream_create("/foo", "PrepData")
-        # Bad layout type
-        with assert_raises(ValueError):
-            db.stream_create("/newton/prep", "NoSuchLayout")
-        db.stream_create("/newton/prep", "PrepData")
-        db.stream_create("/newton/raw", "RawData")
-        db.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
-
-        # Verify we got 3 streams
-        eq_(db.stream_list(), [ ["/newton/prep", "PrepData"],
-                                ["/newton/raw", "RawData"],
-                                ["/newton/zzz/rawnotch", "RawNotchedData"]
-                                ])
-        # Match just one type or one path
-        eq_(db.stream_list(layout="RawData"), [ ["/newton/raw", "RawData"] ])
-        eq_(db.stream_list(path="/newton/raw"), [ ["/newton/raw", "RawData"] ])
-
-        # Verify that columns were made right (pytables specific)
-        if "h5file" in db.data.__dict__:
-            h5file = db.data.h5file
-            eq_(len(h5file.getNode("/newton/prep").cols), 9)
-            eq_(len(h5file.getNode("/newton/raw").cols), 7)
-            eq_(len(h5file.getNode("/newton/zzz/rawnotch").cols), 10)
-            assert(not h5file.getNode("/newton/prep").colindexed["timestamp"])
-            assert(not h5file.getNode("/newton/prep").colindexed["c1"])
-
-        # Set / get metadata
-        eq_(db.stream_get_metadata("/newton/prep"), {})
-        eq_(db.stream_get_metadata("/newton/raw"), {})
-        meta1 = { "description": "The Data",
-                  "v_scale": "1.234" }
-        meta2 = { "description": "The Data" }
-        meta3 = { "v_scale": "1.234" }
-        db.stream_set_metadata("/newton/prep", meta1)
-        db.stream_update_metadata("/newton/prep", {})
-        db.stream_update_metadata("/newton/raw", meta2)
-        db.stream_update_metadata("/newton/raw", meta3)
-        eq_(db.stream_get_metadata("/newton/prep"), meta1)
-        eq_(db.stream_get_metadata("/newton/raw"), meta1)
-
-        # fill in some test coverage for start >= end
-        with assert_raises(nilmdb.server.NilmDBError):
-            db.stream_remove("/newton/prep", 0, 0)
-        with assert_raises(nilmdb.server.NilmDBError):
-            db.stream_remove("/newton/prep", 1, 0)
-        db.stream_remove("/newton/prep", 0, 1)
-
-        db.close()
-
-class TestBlockingServer(object):
-    def setUp(self):
-        self.db = nilmdb.NilmDB(testdb, sync=False)
-
-    def tearDown(self):
-        self.db.close()
-
-    def test_blocking_server(self):
-        # Start web app on a custom port
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = True)
-
-        # Run it
-        event = threading.Event()
-        def run_server():
-            self.server.start(blocking = True, event = event)
-        thread = threading.Thread(target = run_server)
-        thread.start()
-        if not event.wait(timeout = 10):
-            raise AssertionError("server didn't start in 10 seconds")
-
-        # Send request to exit.
-        req = urlopen("http://127.0.0.1:12380/exit/", timeout = 1)
-
-        # Wait for it
-        thread.join()
-
-def geturl(path):
-    req = urlopen("http://127.0.0.1:12380" + path, timeout = 10)
-    return req.read()
-
-def getjson(path):
-    return json.loads(geturl(path))
-
-class TestServer(object):
-
-    def setUp(self):
-        # Start web app on a custom port
-        self.db = nilmdb.NilmDB(testdb, sync=False)
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = False)
-        self.server.start(blocking = False)
-
-    def tearDown(self):
-        # Close web app
-        self.server.stop()
-        self.db.close()
-
-    def test_server(self):
-        # Make sure we can't force an exit, and test other 404 errors
-        for url in [ "/exit", "/", "/favicon.ico" ]:
-            with assert_raises(HTTPError) as e:
-                geturl(url)
-            eq_(e.exception.code, 404)
-
-        # Check version
-        eq_(distutils.version.LooseVersion(getjson("/version")),
-            distutils.version.LooseVersion(nilmdb.__version__))
-
-    def test_stream_list(self):
-        # Known streams that got populated by an earlier test (test_nilmdb)
-        streams = getjson("/stream/list")
-
-        eq_(streams, [
-            ['/newton/prep', 'PrepData'],
-            ['/newton/raw', 'RawData'],
-            ['/newton/zzz/rawnotch', 'RawNotchedData'],
-            ])
-
-        streams = getjson("/stream/list?layout=RawData")
-        eq_(streams, [['/newton/raw', 'RawData']])
-
-        streams = getjson("/stream/list?layout=NoSuchLayout")
-        eq_(streams, [])
-
-
-    def test_stream_metadata(self):
-        with assert_raises(HTTPError) as e:
-            getjson("/stream/get_metadata?path=foo")
-        eq_(e.exception.code, 404)
-
-        data = getjson("/stream/get_metadata?path=/newton/prep")
-        eq_(data, {'description': 'The Data', 'v_scale': '1.234'})
-
-        data = getjson("/stream/get_metadata?path=/newton/prep"
-                       "&key=v_scale")
-        eq_(data, {'v_scale': '1.234'})
-
-        data = getjson("/stream/get_metadata?path=/newton/prep"
-                       "&key=v_scale&key=description")
-        eq_(data, {'description': 'The Data', 'v_scale': '1.234'})
-
-        data = getjson("/stream/get_metadata?path=/newton/prep"
-                       "&key=v_scale&key=foo")
-        eq_(data, {'foo': None, 'v_scale': '1.234'})
-
-        data = getjson("/stream/get_metadata?path=/newton/prep"
-                       "&key=foo")
-        eq_(data, {'foo': None})
-
-
-    def test_insert(self):
-        # GET instead of POST (no body)
-        # (actual POST test is done by client code)
-        with assert_raises(HTTPError) as e:
-            getjson("/stream/insert?path=/newton/prep&start=0&end=0")
-        eq_(e.exception.code, 400)
-