update test times

git-svn-id: https://bucket.mit.edu/svn/nilm/tags/nilmdb-orig-tests@11322 ddd99763-3ecb-0310-9145-efcb8ce7c51f

.coveragerc

@@ -1,11 +0,0 @@
-# -*- conf -*-
-
-[run]
-branch = True
-
-[report]
-exclude_lines =
-	pragma: no cover
-	if 0:
-omit = nilmdb/scripts,nilmdb/_version.py,nilmdb/fsck
-show_missing = True

.gitattributes

@@ -1 +0,0 @@
-nilmdb/_version.py export-subst

.gitignore

@@ -1,24 +0,0 @@
-# Tests
-tests/*testdb/
-.coverage
-db/
-
-# Compiled / cythonized files
-README.html
-docs/*.html
-build/
-*.pyc
-nilmdb/server/interval.c
-nilmdb/server/layout.c
-nilmdb/server/rbtree.c
-*.so
-
-# Setup junk
-dist/
-nilmdb.egg-info/
-venv/
-.eggs/
-
-# Misc
-timeit*out
-

MANIFEST.in

@@ -1,29 +0,0 @@
-# Root
-include README.txt
-include setup.cfg
-include setup.py
-include versioneer.py
-include Makefile
-include .coveragerc
-include .pylintrc
-include requirements.txt
-
-# Cython files -- include .pyx source, but not the generated .c files
-# (Downstream systems must have cython installed in order to build)
-recursive-include nilmdb/server *.pyx *.pyxdep *.pxd
-exclude nilmdb/server/interval.c
-exclude nilmdb/server/rbtree.c
-
-# Version
-include nilmdb/_version.py
-
-# Tests
-recursive-include tests *.py
-recursive-include tests/data *
-include tests/test.order
-
-# Docs
-recursive-include docs Makefile *.md
-
-# Extras
-recursive-include extras *

Makefile

@@ -1,50 +1,2 @@
-# By default, run the tests.
+all:
-all: test
+	nosetests nilmdb/test_interval.py
-
-version:
-	python3 setup.py version
-
-build:
-	python3 setup.py build_ext --inplace
-
-dist: sdist
-sdist:
-	python3 setup.py sdist
-
-install:
-	python3 setup.py install
-
-develop:
-	python3 setup.py develop
-
-docs:
-	make -C docs
-
-ctrl: flake
-flake:
-	flake8 nilmdb
-lint:
-	pylint3 --rcfile=setup.cfg nilmdb
-
-test:
-ifneq ($(INSIDE_EMACS),)
-# Use the slightly more flexible script
-	python3 setup.py build_ext --inplace
-	python3 tests/runtests.py
-else
-# Let setup.py check dependencies, build stuff, and run the test
-	python3 setup.py nosetests
-endif
-
-clean::
-	find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
-	rm -f .coverage
-	rm -rf tests/*testdb*
-	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so
-	make -C docs clean
-
-gitclean::
-	git clean -dXf
-
-.PHONY: all version build dist sdist install docs test
-.PHONY: ctrl lint flake clean gitclean

README.md

@@ -1,40 +0,0 @@
-# nilmdb: Non-Intrusive Load Monitor Database
-by Jim Paris <jim@jtan.com>
-
-NilmDB requires Python 3.8 or newer.
-
-## Prerequisites:
-
-    # Runtime and build environments
-    sudo apt install python3 python3-dev python3-venv python3-pip
-
-    # Create a new Python virtual environment to isolate deps.
-    python3 -m venv ../venv
-    source ../venv/bin/activate   # run "deactivate" to leave
-
-    # Install all Python dependencies
-    pip3 install -r requirements.txt
-
-## Test:
-
-    python3 setup.py nosetests
-
-## Install:
-
-Install it into the virtual environment
-
-    python3 setup.py install
-
-If you want to instead install it system-wide, you will also need to
-install the requirements system-wide:
-
-    sudo pip3 install -r requirements.txt
-    sudo python3 setup.py install
-
-## Usage:
-
-    nilmdb-server --help
-    nilmdb-fsck --help
-    nilmtool --help
-
-See docs/wsgi.md for info on setting up a WSGI application in Apache.

README.txt

@@ -0,0 +1,4 @@
+To install, 
+
+   python seutp.py install
+

bin/nilm-test.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+
+from nilmdb import Interval
+from optparse import OptionParser
+import sys
+
+version = "1.0"
+
+parser = OptionParser()
+parser.add_option("-d", "--db", dest="database", metavar="DATABASE",
+                  help="location of sqlite database")
+parser.add_option("-V", "--version", dest="version", default=False, action="store_true",
+                  help="print version then exit")
+
+(options, args) = parser.parse_args()
+
+if (options.version):
+    print "This script version: " + version
+    sys.exit(0)
+
+if options.database is None:
+    print "Error: database is mandatory"
+    sys.exit(1)
+
+print "Database is " + options.database
+

docs/Makefile

@@ -1,9 +0,0 @@
-ALL_DOCS = $(wildcard *.md)
-
-all: $(ALL_DOCS:.md=.html)
-
-%.html: %.md
-	pandoc -s $< > $@
-
-clean:
-	rm -f *.html

docs/TODO.md

@@ -1,5 +0,0 @@
-- Documentation
-
-- Machine-readable information in OverflowError, parser errors.
-  Maybe subclass `cherrypy.HTTPError` and override `set_response`
-  to add another JSON field?

docs/design.md

@@ -1,469 +0,0 @@
-Structure
----------
-nilmdb.nilmdb is the NILM database interface.  A nilmdb.BulkData
-interface stores data in flat files, and a SQL database tracks
-metadata and ranges.
-
-Access to the nilmdb must be single-threaded.  This is handled with
-the nilmdb.serializer class.  In the future this could probably
-be turned into a per-path serialization.
-
-nilmdb.server is a HTTP server that provides an interface to talk,
-thorugh the serialization layer, to the nilmdb object.
-
-nilmdb.client is a HTTP client that connects to this.
-
-Sqlite performance
-------------------
-
-Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
-takes about 125msec.  sqlite3 will commit transactions at 3 times:
-
-1. explicit con.commit()
-
-2. between a series of DML commands and non-DML commands, e.g.
-   after a series of INSERT, SELECT, but before a CREATE TABLE or
-   PRAGMA.
-
-3. at the end of an explicit transaction, e.g. "with self.con as con:"
-
-To speed up testing, or if this transaction speed becomes an issue,
-the sync=False option to NilmDB will set PRAGMA synchronous=OFF.
-
-
-Inserting streams
------------------
-
-We need to send the contents of "data" as POST.  Do we need chunked
-transfer?
-
-- Don't know the size in advance, so we would need to use chunked if
-  we send the entire thing in one request.
-- But we shouldn't send one chunk per line, so we need to buffer some
-  anyway; why not just make new requests?
-- Consider the infinite-streaming case, we might want to send it
-  immediately?  Not really -- server still should do explicit inserts
-  of fixed-size chunks.
-- Even chunked encoding needs the size of each chunk beforehand, so
-  everything still gets buffered.  Just a tradeoff of buffer size.
-
-Before timestamps are added:
-
-- Raw data is about 440 kB/s    (9 channels)
-- Prep data is about 12.5 kB/s  (1 phase)
-- How do we know how much data to send?
-
-    - Remember that we can only do maybe 8-50 transactions per second on
-      the sqlite database.  So if one block of inserted data is one
-      transaction, we'd need the raw case to be around 64kB per request,
-      ideally more.
-    - Maybe use a range, based on how long it's taking to read the data
-        - If no more data, send it
-        - If data > 1 MB, send it
-    - If more than 10 seconds have elapsed, send it
-    - Should those numbers come from the server?
-
-Converting from ASCII to PyTables:
-
-- For each row getting added, we need to set attributes on a PyTables
-  Row object and call table.append().  This means that there isn't a
-  particularly efficient way of converting from ascii.
-- Could create a function like nilmdb.layout.Layout("foo".fillRow(asciiline)
-    - But this means we're doing parsing on the serialized side
-    - Let's keep parsing on the threaded server side so we can detect
-      errors better, and not block the serialized nilmdb for a slow
-      parsing process.
-- Client sends ASCII data
-- Server converts this ACSII data to a list of values
-    - Maybe:
-
-            # threaded side creates this object
-            parser = nilmdb.layout.Parser("layout_name")
-            # threaded side parses and fills it with data
-            parser.parse(textdata)
-            # serialized side pulls out rows
-            for n in xrange(parser.nrows):
-                parser.fill_row(rowinstance, n)
-                table.append()
-
-
-Inserting streams, inside nilmdb
---------------------------------
-
-- First check that the new stream doesn't overlap.
-    - Get minimum timestamp, maximum timestamp from data parser.
-        - (extend parser to verify monotonicity and track extents)
-    - Get all intervals for this stream in the database
-    - See if new interval overlaps any existing ones
-        - If so, bail
-    - Question: should we cache intervals inside NilmDB?
-        - Assume database is fast for now, and always rebuild fom DB.
-        - Can add a caching layer later if we need to.
-    - `stream_get_ranges(path)` -> return IntervalSet?
-
-Speed
------
-
-- First approach was quadratic.  Adding four hours of data:
-
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/1/raw
-        real    24m31.093s
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/1/raw
-        real    43m44.528s
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-130002 /bpnilm/1/raw
-        real    93m29.713s
-        $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-140003 /bpnilm/1/raw
-        real    166m53.007s
-
-- Disabling pytables indexing didn't help:
-
-        real    31m21.492s
-        real    52m51.963s
-        real    102m8.151s
-        real    176m12.469s
-
-- Server RAM usage is constant.
-
-- Speed problems were due to IntervalSet speed, of parsing intervals
-  from the database and adding the new one each time.
-
-    - First optimization is to cache result of `nilmdb:_get_intervals`,
-      which gives the best speedup.
-
-    - Also switched to internally using bxInterval from bx-python package.
-      Speed of `tests/test_interval:TestIntervalSpeed` is pretty decent
-      and seems to be growing logarithmically now.  About 85μs per insertion
-      for inserting 131k entries.
-
-    - Storing the interval data in SQL might be better, with a scheme like:
-      http://www.logarithmic.net/pfh/blog/01235197474
-
-- Next slowdown target is nilmdb.layout.Parser.parse().
-    - Rewrote parsers using cython and sscanf
-    - Stats (rev 10831), with `_add_interval` disabled
-
-        layout.pyx.Parser.parse:128        6303 sec, 262k calls
-         layout.pyx.parse:63               13913 sec, 5.1g calls
-        numpy:records.py.fromrecords:569   7410 sec, 262k calls
-
-- Probably OK for now.
-
-- After all updates, now takes about 8.5 minutes to insert an hour of
-  data, constant after adding 171 hours (4.9 billion data points)
-
-- Data set size: 98 gigs = 20 bytes per data point.
-  6 uint16 data + 1 uint32 timestamp = 16 bytes per point
-  So compression must be off -- will retry with compression forced on.
-
-IntervalSet speed
------------------
-- Initial implementation was pretty slow, even with binary search in
-  sorted list
-
-- Replaced with bxInterval; now takes about log n time for an insertion
-    - TestIntervalSpeed with range(17,18) and profiling
-        - 85 μs each
-        - 131072 calls to `__iadd__`
-        - 131072 to bx.insert_interval
-        - 131072 to bx.insert:395
-        - 2355835 to bx.insert:106  (18x as many?)
-
-- Tried blist too, worse than bxinterval.
-
-- Might be algorithmic improvements to be made in Interval.py,
-  like in `__and__`
-
-- Replaced again with rbtree.  Seems decent.  Numbers are time per
-  insert for 2**17 insertions, followed by total wall time and RAM
-  usage for running "make test" with `test_rbtree` and `test_interval`
-  with range(5,20):
-    - old values with bxinterval:
-      20.2 μS, total 20 s, 177 MB RAM
-    - rbtree, plain python:
-      97 μS, total 105 s, 846 MB RAM
-    - rbtree converted to cython:
-      26 μS, total 29 s, 320 MB RAM
-    - rbtree and interval converted to cython:
-      8.4 μS, total 12 s, 134 MB RAM
-
-- Would like to move Interval itself back to Python so other
-  non-cythonized code like client code can use it more easily.
-  Testing speed with just `test_interval` being tested, with
-  `range(5,22)`, using `/usr/bin/time -v python tests/runtests.py`,
-  times recorded for 2097152:
-    - 52ae397 (Interval in cython):
-	  12.6133 μs each, ratio 0.866533, total 47 sec, 399 MB RAM
-	- 9759dcf (Interval in python):
-	  21.2937 μs each, ratio 1.462870, total 83 sec, 1107 MB RAM
-  That's a huge difference!  Instead, will keep Interval and DBInterval
-  cythonized inside nilmdb, and just have an additional copy in
-  nilmdb.utils for clients to use.
-
-Layouts
--------
-Current/old design has specific layouts: RawData, PrepData, RawNotchedData.
-Let's get rid of this entirely and switch to simpler data types that are
-just collections and counts of a single type.  We'll still use strings
-to describe them, with format:
-
-    type_count
-
-where type is "uint16", "float32", or "float64", and count is an integer.
-
-nilmdb.layout.named() will parse these strings into the appropriate
-handlers.  For compatibility:
-
-    "RawData" == "uint16_6"
-    "RawNotchedData" == "uint16_9"
-    "PrepData" == "float32_8"
-
-
-BulkData design
----------------
-
-BulkData is a custom bulk data storage system that was written to
-replace PyTables.  The general structure is a `data` subdirectory in
-the main NilmDB directory.  Within `data`, paths are created for each
-created stream.  These locations are called tables.  For example,
-tables might be located at
-
-    nilmdb/data/newton/raw/
-    nilmdb/data/newton/prep/
-    nilmdb/data/cottage/raw/
-
-Each table contains:
-
-- An unchanging `_format` file (Python pickle format) that describes
-  parameters of how the data is broken up, like files per directory,
-  rows per file, and the binary data format
-
-- Hex named subdirectories `("%04x", although more than 65536 can exist)`
-
-- Hex named files within those subdirectories, like:
-
-        /nilmdb/data/newton/raw/000b/010a
-
-    The data format of these files is raw binary, interpreted by the
-    Python `struct` module according to the format string in the
-    `_format` file.
-
-- Same as above, with `.removed` suffix, is an optional file (Python
-  pickle format) containing a list of row numbers that have been
-  logically removed from the file.  If this range covers the entire
-  file, the entire file will be removed.
-
-- Note that the `bulkdata.nrows` variable is calculated once in
-  `BulkData.__init__()`, and only ever incremented during use.  Thus,
-  even if all data is removed, `nrows` can remain high.  However, if
-  the server is restarted, the newly calculated `nrows` may be lower
-  than in a previous run due to deleted data.  To be specific, this
-  sequence of events:
-
-    - insert data
-    - remove all data
-    - insert data
-
-    will result in having different row numbers in the database, and
-    differently numbered files on the filesystem, than the sequence:
-
-    - insert data
-    - remove all data
-    - restart server
-    - insert data
-
-    This is okay!  Everything should remain consistent both in the
-    `BulkData` and `NilmDB`.  Not attempting to readjust `nrows` during
-    deletion makes the code quite a bit simpler.
-
-- Similarly, data files are never truncated shorter.  Removing data
-  from the end of the file will not shorten it; it will only be
-  deleted when it has been fully filled and all of the data has been
-  subsequently removed.
-
-
-Rocket
-------
-
-Original design had the nilmdb.nilmdb thread (through bulkdata)
-convert from on-disk layout to a Python list, and then the
-nilmdb.server thread (from cherrypy) converts to ASCII.  For at least
-the extraction side of things, it's easy to pass the bulkdata a layout
-name instead, and have it convert directly from on-disk to ASCII
-format, because this conversion can then be shoved into a C module.
-This module, which provides a means for converting directly from
-on-disk format to ASCII or Python lists, is the "rocket" interface.
-Python is still used to manage the files and figure out where the
-data should go; rocket just puts binary data directly in or out of
-those files at specified locations.
-
-Before rocket, testing speed with uint16_6 data, with an end-to-end
-test (extracting data with nilmtool):
-
-- insert: 65 klines/sec
-- extract: 120 klines/sec
-
-After switching to the rocket design, but using the Python version
-(pyrocket):
-
-- insert: 57 klines/sec
-- extract: 120 klines/sec
-
-After switching to a C extension module (rocket.c)
-
-- insert: 74 klines/sec through insert.py; 99.6 klines/sec through nilmtool
-- extract: 335 klines/sec
-
-After client block updates (described below):
-
-- insert: 180 klines/sec through nilmtool (pre-timestamped)
-- extract: 390 klines/sec through nilmtool
-
-Using "insert --timestamp" or "extract --bare" cuts the speed in half.
-
-Blocks versus lines
--------------------
-
-Generally want to avoid parsing the bulk of the data as lines if
-possible, and transfer things in bigger blocks at once.
-
-Current places where we use lines:
-
-- All data returned by `client.stream_extract`, since it comes from
-  `httpclient.get_gen`, which iterates over lines.  Not sure if this
-  should be changed, because a `nilmtool extract` is just about the
-  same speed as `curl -q .../stream/extract`!
-
-- `client.StreamInserter.insert_iter` and
-  `client.StreamInserter.insert_line`, which should probably get
-  replaced with block versions.  There's no real need to keep
-  updating the timestamp every time we get a new line of data.
-
-  - Finished.  Just a single insert() that takes any length string and
-    does very little processing until it's time to send it to the
-	server.
-
-Timestamps
-----------
-
-Timestamps are currently double-precision floats (64 bit).  Since the
-mantissa is 53-bit, this can only represent about 15-17 significant
-figures, and microsecond Unix timestamps like 1222333444.000111 are
-already 16 significant figures.  Rounding is therefore an issue;
-it's hard to sure that converting from ASCII, then back to ASCII,
-will always give the same result.
-
-Also, if the client provides a floating point value like 1.9999999999,
-we need to be careful that we don't store it as 1.9999999999 but later
-print it as 2.000000, because then round-trips change the data.
-
-Possible solutions:
-
-- When the client provides a floating point value to the server,
-  always round to the 6th decimal digit before verifying & storing.
-  Good for compatibility and simplicity.  But still might have rounding
-  issues, and clients will also need to round when doing their own
-  verification.  Having every piece of code need to know which digit
-  to round at is not ideal.
-
-- Always store int64 timestamps on the server, representing
-  microseconds since epoch.  int64 timestamps are used in all HTTP
-  parameters, in insert/extract ASCII strings, client API, commandline
-  raw timestamps, etc.  Pretty big change.
-
-  This is what we'll go with...
-
-  - Client programs that interpret the timestamps as doubles instead
-    of ints will remain accurate until 2^53 microseconds, or year
-    2255.
-
-  - On insert, maybe it's OK to send floating point microsecond values
-    (1234567890123456.0), just to cope with clients that want to print
-    everything as a double.  Server could try parsing as int64, and if
-    that fails, parse as double and truncate to int64.  However, this
-    wouldn't catch imprecise inputs like "1.23456789012e+15".  But
-    maybe that can just be ignored; it's likely to cause a
-    non-monotonic error at the client.
-
-  - Timestamps like 1234567890.123456 never show up anywhere, except
-    for interfacing to datetime_tz etc.  Command line "raw timestamps"
-    are always printed as int64 values, and a new format
-    "@1234567890123456" is added to the parser for specifying them
-    exactly.
-
-Binary interface
-----------------
-
-The ASCII interface is too slow for high-bandwidth processing, like
-sinefits, prep, etc.  A binary interface was added so that you can
-extract the raw binary out of the bulkdata storage.  This binary is
-a little-endian format, e.g. in C a uint16_6 stream would be:
-
-    #include <endian.h>
-    #include <stdint.h>
-    struct {
-        int64_t timestamp_le;
-        uint16_t data_le[6];
-    } __attribute__((packed));
-
-Remember to byteswap (with e.g. `letoh` in C)!
-
-This interface is used by the new `nilmdb.client.numpyclient.NumpyClient`
-class, which is a subclass of the normal `nilmcb.client.client.Client`
-and has all of the same functions.  It adds three new functions:
-
-- `stream_extract_numpy` to extract data as a Numpy array
-
-- `stream_insert_numpy` to insert data as a Numpy array
-
-- `stream_insert_numpy_context` is the context manager for
-  incrementally inserting data
-
-It is significantly faster!  It is about 20 times faster to decimate a
-stream with `nilm-decimate` when the filter code is using the new
-binary/numpy interface.
-
-
-WSGI interface & chunked requests
----------------------------------
-
-mod_wsgi requires "WSGIChunkedRequest On" to handle
-"Transfer-encoding: Chunked" requests.  However, `/stream/insert`
-doesn't handle this correctly right now, because:
-
-- The `cherrypy.request.body.read()` call needs to be fixed for chunked requests
-
-- We don't want to just buffer endlessly in the server, and it will
-  require some thought on how to handle data in chunks (what to do about
-  interval endpoints).
-
-It is probably better to just keep the endpoint management on the client
-side, so leave "WSGIChunkedRequest off" for now.
-
-
-Unicode & character encoding
-----------------------------
-
-Stream data is passed back and forth as raw `bytes` objects in most
-places, including the `nilmdb.client` and command-line interfaces.
-This is done partially for performance reasons, and partially to
-support the binary insert/extract options, where character-set encoding
-would not apply.
-
-For the HTTP server, the raw bytes transferred over HTTP are interpreted
-as follows:
-- For `/stream/insert`, the client-provided `Content-Type` is ignored,
-  and the data is read as if it were `application/octet-stream`.
-- For `/stream/extract`, the returned data is `application/octet-stream`.
-- All other endpoints communicate via JSON, which is specified to always
-  be encoded as UTF-8.  This includes:
-    - `/version`
-    - `/dbinfo`
-    - `/stream/list`
-    - `/stream/create`
-    - `/stream/destroy`
-    - `/stream/rename`
-    - `/stream/get_metadata`
-    - `/stream/set_metadata`
-    - `/stream/update_metadata`
-    - `/stream/remove`
-    - `/stream/intervals`

docs/wsgi.md

@@ -1,32 +0,0 @@
-WSGI Application in Apache
---------------------------
-
-Install `apache2` and `libapache2-mod-wsgi`
-
-We'll set up the database server at URL `http://myhost.com/nilmdb`.
-The database will be stored in `/home/nilm/db`, and the process will
-run as user `nilm`, group `nilm`.
-
-First, create a WSGI script `/home/nilm/nilmdb.wsgi` containing:
-
-    import nilmdb.server
-    application = nilmdb.server.wsgi_application("/home/nilm/db", "/nilmdb")
-
-The first parameter is the local filesystem path, and the second
-parameter is the path part of the URL.
-
-Then, set up Apache with a configuration like:
-
-    <VirtualHost>
-        WSGIScriptAlias /nilmdb /home/nilm/nilmdb.wsgi
-        WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilm group=nilm
-        <Location /nilmdb>
-            WSGIProcessGroup nilmdb-procgroup
-            WSGIApplicationGroup nilmdb-appgroup
-
-            # Access control example:
-            Order deny,allow
-            Deny from all
-            Allow from 1.2.3.4
-        </Location>
-    </VirtualHost>

extras/fix-oversize-files.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import pickle
-import argparse
-import fcntl
-import re
-from nilmdb.client.numpyclient import layout_to_dtype
-
-parser = argparse.ArgumentParser(
-    description = """
-Fix database corruption where binary writes caused too much data to be
-written to the file.  Truncates files to the correct length.  This was
-fixed by b98ff1331a515ad47fd3203615e835b529b039f9.
-""")
-parser.add_argument("path", action="store", help='Database root path')
-parser.add_argument("-y", "--yes", action="store_true", help='Fix them')
-args = parser.parse_args()
-
-lock = os.path.join(args.path, "data.lock")
-with open(lock, "w") as f:
-    fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
-
-    fix = {}
-
-    for (path, dirs, files) in os.walk(args.path):
-        if "_format" in files:
-            with open(os.path.join(path, "_format")) as format:
-                fmt = pickle.load(format)
-                rowsize = layout_to_dtype(fmt["layout"]).itemsize
-                maxsize = rowsize * fmt["rows_per_file"]
-                fix[path] = maxsize
-                if maxsize < 128000000: # sanity check
-                    raise Exception("bad maxsize " + str(maxsize))
-
-    for fixpath in fix:
-        for (path, dirs, files) in os.walk(fixpath):
-            for fn in files:
-                if not re.match("^[0-9a-f]{4,}$", fn):
-                    continue
-                fn = os.path.join(path, fn)
-                size = os.path.getsize(fn)
-                maxsize = fix[fixpath]
-                if size > maxsize:
-                    diff = size - maxsize
-                    print(diff, "too big:", fn)
-                    if args.yes:
-                        with open(fn, "a+") as dbfile:
-                            dbfile.truncate(maxsize)

extras/nilmtool-bash-completion.sh

@@ -1,20 +0,0 @@
-# To enable bash completion:
-#
-# 1. Ensure python-argcomplete is installed:
-#       pip install argcomplete
-# 2. Source this file:
-#       . nilmtool-bash-completion.sh
-
-_nilmtool_argcomplete() {
-    local IFS=$(printf "\013")
-    COMPREPLY=( $(IFS="$IFS" \
-                  COMP_LINE="$COMP_LINE" \
-	          COMP_WORDBREAKS="$COMP_WORDBREAKS" \
-                  COMP_POINT="$COMP_POINT" \
-                  _ARGCOMPLETE=1 \
-                  "$1" 8>&1 9>&2 1>/dev/null 2>/dev/null) )
-    if [[ $? != 0 ]]; then
-        unset COMPREPLY
-    fi
-}
-complete -o nospace -F _nilmtool_argcomplete nilmtool

nilmdb/__init__.py

@@ -1,5 +1,2 @@
-"""Main NilmDB import"""
+from nilmdb.interval import *
-
+from nilmdb.fileinterval import *
-from ._version import get_versions
-__version__ = get_versions()['version']
-del get_versions

nilmdb/_version.py

@@ -1,520 +0,0 @@
-
-# This file helps to compute a version number in source trees obtained from
-# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (built by setup.py sdist) and build
-# directories (produced by setup.py build) will contain a much shorter file
-# that just contains the computed version number.
-
-# This file is released into the public domain. Generated by
-# versioneer-0.18 (https://github.com/warner/python-versioneer)
-
-"""Git implementation of _version.py."""
-
-import errno
-import os
-import re
-import subprocess
-import sys
-
-
-def get_keywords():
-    """Get the keywords needed to look up the version information."""
-    # these strings will be replaced by git during git-archive.
-    # setup.py/versioneer.py will grep for the variable names, so they must
-    # each be defined on a line of their own. _version.py will just call
-    # get_keywords().
-    git_refnames = "$Format:%d$"
-    git_full = "$Format:%H$"
-    git_date = "$Format:%ci$"
-    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
-    return keywords
-
-
-class VersioneerConfig:
-    """Container for Versioneer configuration parameters."""
-
-
-def get_config():
-    """Create, populate and return the VersioneerConfig() object."""
-    # these strings are filled in when 'setup.py versioneer' creates
-    # _version.py
-    cfg = VersioneerConfig()
-    cfg.VCS = "git"
-    cfg.style = "pep440"
-    cfg.tag_prefix = "nilmdb-"
-    cfg.parentdir_prefix = "nilmdb-"
-    cfg.versionfile_source = "nilmdb/_version.py"
-    cfg.verbose = False
-    return cfg
-
-
-class NotThisMethod(Exception):
-    """Exception raised if a method is not valid for the current scenario."""
-
-
-LONG_VERSION_PY = {}
-HANDLERS = {}
-
-
-def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
-    def decorate(f):
-        """Store f in HANDLERS[vcs][method]."""
-        if vcs not in HANDLERS:
-            HANDLERS[vcs] = {}
-        HANDLERS[vcs][method] = f
-        return f
-    return decorate
-
-
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
-                env=None):
-    """Call the given command(s)."""
-    assert isinstance(commands, list)
-    p = None
-    for c in commands:
-        try:
-            dispcmd = str([c] + args)
-            # remember shell=False, so use git.cmd on windows, not just git
-            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
-                                 stdout=subprocess.PIPE,
-                                 stderr=(subprocess.PIPE if hide_stderr
-                                         else None))
-            break
-        except EnvironmentError:
-            e = sys.exc_info()[1]
-            if e.errno == errno.ENOENT:
-                continue
-            if verbose:
-                print("unable to run %s" % dispcmd)
-                print(e)
-            return None, None
-    else:
-        if verbose:
-            print("unable to find command, tried %s" % (commands,))
-        return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % dispcmd)
-            print("stdout was %s" % stdout)
-        return None, p.returncode
-    return stdout, p.returncode
-
-
-def versions_from_parentdir(parentdir_prefix, root, verbose):
-    """Try to determine the version from the parent directory name.
-
-    Source tarballs conventionally unpack into a directory that includes both
-    the project name and a version string. We will also support searching up
-    two directory levels for an appropriately named parent directory
-    """
-    rootdirs = []
-
-    for i in range(3):
-        dirname = os.path.basename(root)
-        if dirname.startswith(parentdir_prefix):
-            return {"version": dirname[len(parentdir_prefix):],
-                    "full-revisionid": None,
-                    "dirty": False, "error": None, "date": None}
-        else:
-            rootdirs.append(root)
-            root = os.path.dirname(root)  # up a level
-
-    if verbose:
-        print("Tried directories %s but none started with prefix %s" %
-              (str(rootdirs), parentdir_prefix))
-    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
-
-
-@register_vcs_handler("git", "get_keywords")
-def git_get_keywords(versionfile_abs):
-    """Extract version information from the given file."""
-    # the code embedded in _version.py can just fetch the value of these
-    # keywords. When used from setup.py, we don't want to import _version.py,
-    # so we do it with a regexp instead. This function is not used from
-    # _version.py.
-    keywords = {}
-    try:
-        f = open(versionfile_abs, "r")
-        for line in f.readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["full"] = mo.group(1)
-            if line.strip().startswith("git_date ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["date"] = mo.group(1)
-        f.close()
-    except EnvironmentError:
-        pass
-    return keywords
-
-
-@register_vcs_handler("git", "keywords")
-def git_versions_from_keywords(keywords, tag_prefix, verbose):
-    """Get version information from git keywords."""
-    if not keywords:
-        raise NotThisMethod("no keywords at all, weird")
-    date = keywords.get("date")
-    if date is not None:
-        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
-        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
-        # -like" string, which we must then edit to make compliant), because
-        # it's been around since git-1.5.3, and it's too difficult to
-        # discover which version we're using, or to work around using an
-        # older one.
-        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
-    refnames = keywords["refnames"].strip()
-    if refnames.startswith("$Format"):
-        if verbose:
-            print("keywords are unexpanded, not using")
-        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
-    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
-    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
-    TAG = "tag: "
-    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
-    if not tags:
-        # Either we're using git < 1.8.3, or there really are no tags. We use
-        # a heuristic: assume all version tags have a digit. The old git %d
-        # expansion behaves like git log --decorate=short and strips out the
-        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
-        # between branches and tags. By ignoring refnames without digits, we
-        # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
-        tags = set([r for r in refs if re.search(r'\d', r)])
-        if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
-    if verbose:
-        print("likely tags: %s" % ",".join(sorted(tags)))
-    for ref in sorted(tags):
-        # sorting will prefer e.g. "2.0" over "2.0rc1"
-        if ref.startswith(tag_prefix):
-            r = ref[len(tag_prefix):]
-            if verbose:
-                print("picking %s" % r)
-            return {"version": r,
-                    "full-revisionid": keywords["full"].strip(),
-                    "dirty": False, "error": None,
-                    "date": date}
-    # no suitable tags, so version is "0+unknown", but full hex is still there
-    if verbose:
-        print("no suitable tags, using unknown + full revision id")
-    return {"version": "0+unknown",
-            "full-revisionid": keywords["full"].strip(),
-            "dirty": False, "error": "no suitable tags", "date": None}
-
-
-@register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
-    """Get version from 'git describe' in the root of the source tree.
-
-    This only gets called if the git-archive 'subst' keywords were *not*
-    expanded, and _version.py hasn't already been rewritten with a short
-    version string, meaning we're inside a checked out source tree.
-    """
-    GITS = ["git"]
-    if sys.platform == "win32":
-        GITS = ["git.cmd", "git.exe"]
-
-    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
-                          hide_stderr=True)
-    if rc != 0:
-        if verbose:
-            print("Directory %s not under git control" % root)
-        raise NotThisMethod("'git rev-parse --git-dir' returned error")
-
-    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
-    # if there isn't one, this yields HEX[-dirty] (no NUM)
-    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
-                                          "--always", "--long",
-                                          "--match", "%s*" % tag_prefix],
-                                   cwd=root)
-    # --long was added in git-1.5.5
-    if describe_out is None:
-        raise NotThisMethod("'git describe' failed")
-    describe_out = describe_out.strip()
-    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
-    if full_out is None:
-        raise NotThisMethod("'git rev-parse' failed")
-    full_out = full_out.strip()
-
-    pieces = {}
-    pieces["long"] = full_out
-    pieces["short"] = full_out[:7]  # maybe improved later
-    pieces["error"] = None
-
-    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
-    # TAG might have hyphens.
-    git_describe = describe_out
-
-    # look for -dirty suffix
-    dirty = git_describe.endswith("-dirty")
-    pieces["dirty"] = dirty
-    if dirty:
-        git_describe = git_describe[:git_describe.rindex("-dirty")]
-
-    # now we have TAG-NUM-gHEX or HEX
-
-    if "-" in git_describe:
-        # TAG-NUM-gHEX
-        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
-        if not mo:
-            # unparseable. Maybe git-describe is misbehaving?
-            pieces["error"] = ("unable to parse git-describe output: '%s'"
-                               % describe_out)
-            return pieces
-
-        # tag
-        full_tag = mo.group(1)
-        if not full_tag.startswith(tag_prefix):
-            if verbose:
-                fmt = "tag '%s' doesn't start with prefix '%s'"
-                print(fmt % (full_tag, tag_prefix))
-            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
-                               % (full_tag, tag_prefix))
-            return pieces
-        pieces["closest-tag"] = full_tag[len(tag_prefix):]
-
-        # distance: number of commits since tag
-        pieces["distance"] = int(mo.group(2))
-
-        # commit: short hex revision ID
-        pieces["short"] = mo.group(3)
-
-    else:
-        # HEX: no tags
-        pieces["closest-tag"] = None
-        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
-                                    cwd=root)
-        pieces["distance"] = int(count_out)  # total number of commits
-
-    # commit date: see ISO-8601 comment in git_versions_from_keywords()
-    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
-                       cwd=root)[0].strip()
-    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
-
-    return pieces
-
-
-def plus_or_dot(pieces):
-    """Return a + if we don't already have one, else return a ."""
-    if "+" in pieces.get("closest-tag", ""):
-        return "."
-    return "+"
-
-
-def render_pep440(pieces):
-    """Build up version string, with post-release "local version identifier".
-
-    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
-    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
-
-    Exceptions:
-    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += plus_or_dot(pieces)
-            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
-            if pieces["dirty"]:
-                rendered += ".dirty"
-    else:
-        # exception #1
-        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
-                                          pieces["short"])
-        if pieces["dirty"]:
-            rendered += ".dirty"
-    return rendered
-
-
-def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
-
-    Exceptions:
-    1: no tags. 0.post.devDISTANCE
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += ".post.dev%d" % pieces["distance"]
-    else:
-        # exception #1
-        rendered = "0.post.dev%d" % pieces["distance"]
-    return rendered
-
-
-def render_pep440_post(pieces):
-    """TAG[.postDISTANCE[.dev0]+gHEX] .
-
-    The ".dev0" means dirty. Note that .dev0 sorts backwards
-    (a dirty tree will appear "older" than the corresponding clean one),
-    but you shouldn't be releasing software with -dirty anyways.
-
-    Exceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-            rendered += plus_or_dot(pieces)
-            rendered += "g%s" % pieces["short"]
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-        rendered += "+g%s" % pieces["short"]
-    return rendered
-
-
-def render_pep440_old(pieces):
-    """TAG[.postDISTANCE[.dev0]] .
-
-    The ".dev0" means dirty.
-
-    Eexceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-    return rendered
-
-
-def render_git_describe(pieces):
-    """TAG[-DISTANCE-gHEX][-dirty].
-
-    Like 'git describe --tags --dirty --always'.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render_git_describe_long(pieces):
-    """TAG-DISTANCE-gHEX[-dirty].
-
-    Like 'git describe --tags --dirty --always -long'.
-    The distance/hash is unconditional.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render(pieces, style):
-    """Render the given version pieces into the requested style."""
-    if pieces["error"]:
-        return {"version": "unknown",
-                "full-revisionid": pieces.get("long"),
-                "dirty": None,
-                "error": pieces["error"],
-                "date": None}
-
-    if not style or style == "default":
-        style = "pep440"  # the default
-
-    if style == "pep440":
-        rendered = render_pep440(pieces)
-    elif style == "pep440-pre":
-        rendered = render_pep440_pre(pieces)
-    elif style == "pep440-post":
-        rendered = render_pep440_post(pieces)
-    elif style == "pep440-old":
-        rendered = render_pep440_old(pieces)
-    elif style == "git-describe":
-        rendered = render_git_describe(pieces)
-    elif style == "git-describe-long":
-        rendered = render_git_describe_long(pieces)
-    else:
-        raise ValueError("unknown style '%s'" % style)
-
-    return {"version": rendered, "full-revisionid": pieces["long"],
-            "dirty": pieces["dirty"], "error": None,
-            "date": pieces.get("date")}
-
-
-def get_versions():
-    """Get version information or return default if unable to do so."""
-    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
-    # __file__, we can work backwards from there to the root. Some
-    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
-    # case we can only use expanded keywords.
-
-    cfg = get_config()
-    verbose = cfg.verbose
-
-    try:
-        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
-                                          verbose)
-    except NotThisMethod:
-        pass
-
-    try:
-        root = os.path.realpath(__file__)
-        # versionfile_source is the relative path from the top of the source
-        # tree (where the .git directory might live) to this file. Invert
-        # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):
-            root = os.path.dirname(root)
-    except NameError:
-        return {"version": "0+unknown", "full-revisionid": None,
-                "dirty": None,
-                "error": "unable to find root of source tree",
-                "date": None}
-
-    try:
-        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
-        return render(pieces, cfg.style)
-    except NotThisMethod:
-        pass
-
-    try:
-        if cfg.parentdir_prefix:
-            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
-    except NotThisMethod:
-        pass
-
-    return {"version": "0+unknown", "full-revisionid": None,
-            "dirty": None,
-            "error": "unable to compute version", "date": None}

nilmdb/client/__init__.py

@@ -1,4 +0,0 @@
-"""nilmdb.client"""
-
-from nilmdb.client.client import Client
-from nilmdb.client.errors import ClientError, ServerError, Error

nilmdb/client/client.py

@@ -1,477 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Class for performing HTTP client requests via libcurl"""
-
-import json
-import contextlib
-
-import nilmdb.utils
-import nilmdb.client.httpclient
-from nilmdb.client.errors import ClientError
-from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
-
-
-def extract_timestamp(line):
-    """Extract just the timestamp from a line of data text"""
-    return string_to_timestamp(line.split()[0])
-
-
-class Client():
-    """Main client interface to the Nilm database."""
-
-    def __init__(self, url, post_json=False):
-        """Initialize client with given URL.  If post_json is true,
-        POST requests are sent with Content-Type 'application/json'
-        instead of the default 'x-www-form-urlencoded'."""
-        self.http = nilmdb.client.httpclient.HTTPClient(url, post_json)
-        self.post_json = post_json
-
-    # __enter__/__exit__ allow this class to be a context manager
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-
-    def _json_post_param(self, data):
-        """Return compact json-encoded version of parameter"""
-        if self.post_json:
-            # If we're posting as JSON, we don't need to encode it further here
-            return data
-        return json.dumps(data, separators=(',', ':'))
-
-    def close(self):
-        """Close the connection; safe to call multiple times"""
-        self.http.close()
-
-    def geturl(self):
-        """Return the URL we're using"""
-        return self.http.baseurl
-
-    def version(self):
-        """Return server version"""
-        return self.http.get("version")
-
-    def dbinfo(self):
-        """Return server database info (path, size, free space)
-        as a dictionary."""
-        return self.http.get("dbinfo")
-
-    def stream_list(self, path=None, layout=None, extended=False):
-        """Return a sorted list of [path, layout] lists.  If 'path' or
-        'layout' are specified, only return streams that match those
-        exact values.  If 'extended' is True, the returned lists have
-        extended info, e.g.: [path, layout, extent_min, extent_max,
-        total_rows, total_seconds."""
-        params = {}
-        if path is not None:
-            params["path"] = path
-        if layout is not None:
-            params["layout"] = layout
-        if extended:
-            params["extended"] = 1
-        streams = self.http.get("stream/list", params)
-        return nilmdb.utils.sort.sort_human(streams, key=lambda s: s[0])
-
-    def stream_get_metadata(self, path, keys=None):
-        """Get stream metadata"""
-        params = {"path": path}
-        if keys is not None:
-            params["key"] = keys
-        return self.http.get("stream/get_metadata", params)
-
-    def stream_set_metadata(self, path, data):
-        """Set stream metadata from a dictionary, replacing all existing
-        metadata."""
-        params = {
-            "path": path,
-            "data": self._json_post_param(data)
-        }
-        return self.http.post("stream/set_metadata", params)
-
-    def stream_update_metadata(self, path, data):
-        """Update stream metadata from a dictionary"""
-        params = {
-            "path": path,
-            "data": self._json_post_param(data)
-        }
-        return self.http.post("stream/update_metadata", params)
-
-    def stream_create(self, path, layout):
-        """Create a new stream"""
-        params = {
-            "path": path,
-            "layout": layout
-        }
-        return self.http.post("stream/create", params)
-
-    def stream_destroy(self, path):
-        """Delete stream.  Fails if any data is still present."""
-        params = {
-            "path": path
-        }
-        return self.http.post("stream/destroy", params)
-
-    def stream_rename(self, oldpath, newpath):
-        """Rename a stream."""
-        params = {
-            "oldpath": oldpath,
-            "newpath": newpath
-        }
-        return self.http.post("stream/rename", params)
-
-    def stream_remove(self, path, start=None, end=None):
-        """Remove data from the specified time range"""
-        params = {
-            "path": path
-        }
-        if start is not None:
-            params["start"] = timestamp_to_string(start)
-        if end is not None:
-            params["end"] = timestamp_to_string(end)
-        total = 0
-        for count in self.http.post_gen("stream/remove", params):
-            total += int(count)
-        return total
-
-    @contextlib.contextmanager
-    def stream_insert_context(self, path, start=None, end=None):
-        """Return a context manager that allows data to be efficiently
-        inserted into a stream in a piecewise manner.  Data is
-        provided as ASCII lines, and is aggregated and sent to the
-        server in larger or smaller chunks as necessary.  Data lines
-        must match the database layout for the given path, and end
-        with a newline.
-
-        Example:
-          with client.stream_insert_context('/path', start, end) as ctx:
-            ctx.insert('1234567890000000 1 2 3 4\\n')
-            ctx.insert('1234567891000000 1 2 3 4\\n')
-
-        For more details, see help for nilmdb.client.client.StreamInserter
-
-        This may make multiple requests to the server, if the data is
-        large enough or enough time has passed between insertions.
-        """
-        ctx = StreamInserter(self, path, start, end)
-        yield ctx
-        ctx.finalize()
-        ctx.destroy()
-
-    def stream_insert(self, path, data, start=None, end=None):
-        """Insert rows of data into a stream.  data should be a string
-        or iterable that provides ASCII data that matches the database
-        layout for path.  Data is passed through stream_insert_context,
-        so it will be broken into reasonably-sized chunks and
-        start/end will be deduced if missing."""
-        with self.stream_insert_context(path, start, end) as ctx:
-            if isinstance(data, bytes):
-                ctx.insert(data)
-            else:
-                for chunk in data:
-                    ctx.insert(chunk)
-        return ctx.last_response
-
-    def stream_insert_block(self, path, data, start, end, binary=False):
-        """Insert a single fixed block of data into the stream.  It is
-        sent directly to the server in one block with no further
-        processing.
-
-        If 'binary' is True, provide raw binary data in little-endian
-        format matching the path layout, including an int64 timestamp.
-        Otherwise, provide ASCII data matching the layout."""
-        params = {
-            "path": path,
-            "start": timestamp_to_string(start),
-            "end": timestamp_to_string(end),
-        }
-        if binary:
-            params["binary"] = 1
-        return self.http.put("stream/insert", data, params)
-
-    def stream_intervals(self, path, start=None, end=None, diffpath=None):
-        """
-        Return a generator that yields each stream interval.
-
-        If 'diffpath' is not None, yields only interval ranges that are
-        present in 'path' but not in 'diffpath'.
-        """
-        params = {
-            "path": path
-        }
-        if diffpath is not None:
-            params["diffpath"] = diffpath
-        if start is not None:
-            params["start"] = timestamp_to_string(start)
-        if end is not None:
-            params["end"] = timestamp_to_string(end)
-        return self.http.get_gen("stream/intervals", params)
-
-    def stream_extract(self, path, start=None, end=None,
-                       count=False, markup=False, binary=False):
-        """
-        Extract data from a stream.  Returns a generator that yields
-        lines of ASCII-formatted data that matches the database
-        layout for the given path.
-
-        If 'count' is True, return a count of matching data points
-        rather than the actual data.  The output format is unchanged.
-
-        If 'markup' is True, include comments in the returned data
-        that indicate interval starts and ends.
-
-        If 'binary' is True, return chunks of raw binary data, rather
-        than lines of ASCII-formatted data.  Raw binary data is
-        little-endian and matches the database types (including an
-        int64 timestamp).
-        """
-        params = {
-            "path": path,
-        }
-        if start is not None:
-            params["start"] = timestamp_to_string(start)
-        if end is not None:
-            params["end"] = timestamp_to_string(end)
-        if count:
-            params["count"] = 1
-        if markup:
-            params["markup"] = 1
-        if binary:
-            params["binary"] = 1
-        return self.http.get_gen("stream/extract", params, binary=binary)
-
-    def stream_count(self, path, start=None, end=None):
-        """
-        Return the number of rows of data in the stream that satisfy
-        the given timestamps.
-        """
-        counts = list(self.stream_extract(path, start, end, count=True))
-        return int(counts[0])
-
-
-class StreamInserter():
-    """Object returned by stream_insert_context() that manages
-    the insertion of rows of data into a particular path.
-
-    The basic data flow is that we are filling a contiguous interval
-    on the server, with no gaps, that extends from timestamp 'start'
-    to timestamp 'end'.  Data timestamps satisfy 'start <= t < end'.
-
-    Data is provided to .insert() as ASCII formatted data separated by
-    newlines.  The chunks of data passed to .insert() do not need to
-    match up with the newlines; less or more than one line can be passed.
-
-    1. The first inserted line begins a new interval that starts at
-    'start'.  If 'start' is not given, it is deduced from the first
-    line's timestamp.
-
-    2. Subsequent lines go into the same contiguous interval.  As lines
-    are inserted, this routine may make multiple insertion requests to
-    the server, but will structure the timestamps to leave no gaps.
-
-    3. The current contiguous interval can be completed by manually
-    calling .finalize(), which the context manager will also do
-    automatically.  This will send any remaining data to the server,
-    using the 'end' timestamp to end the interval.  If no 'end'
-    was provided, it is deduced from the last timestamp seen,
-    plus a small delta.
-
-    After a .finalize(), inserting new data goes back to step 1.
-
-    .update_start() can be called before step 1 to change the start
-    time for the interval.  .update_end() can be called before step 3
-    to change the end time for the interval.
-    """
-
-    # See design.md for a discussion of how much data to send.  This
-    # is a soft limit -- we might send up to twice as much or so
-    _max_data = 2 * 1024 * 1024
-    _max_data_after_send = 64 * 1024
-
-    def __init__(self, client, path, start, end):
-        """'client' is the client object.  'path' is the database
-        path to insert to.  'start' and 'end' are used for the first
-        contiguous interval and may be None."""
-        self.last_response = None
-
-        self._client = client
-        self._path = path
-
-        # Start and end for the overall contiguous interval we're
-        # filling
-        self._interval_start = start
-        self._interval_end = end
-
-        # Current data we're building up to send.  Each string
-        # goes into the array, and gets joined all at once.
-        self._block_data = []
-        self._block_len = 0
-
-        self.destroyed = False
-
-    def destroy(self):
-        """Ensure this object can't be used again without raising
-        an error"""
-        def error(*args, **kwargs):
-            raise Exception("don't reuse this context object")
-        self._send_block = self.insert = self.finalize = self.send = error
-
-    def insert(self, data):
-        """Insert a chunk of ASCII formatted data in string form.  The
-        overall data must consist of lines terminated by '\\n'."""
-        length = len(data)
-        maxdata = self._max_data
-
-        if length > maxdata:
-            # This could make our buffer more than twice what we
-            # wanted to send, so split it up.  This is a bit
-            # inefficient, but the user really shouldn't be providing
-            # this much data at once.
-            for cut in range(0, length, maxdata):
-                self.insert(data[cut:(cut + maxdata)])
-            return
-
-        # Append this string to our list
-        self._block_data.append(data)
-        self._block_len += length
-
-        # Send the block once we have enough data
-        if self._block_len >= maxdata:
-            self._send_block(final=False)
-            if self._block_len >= self._max_data_after_send:
-                raise ValueError("too much data left over after trying"
-                                 " to send intermediate block; is it"
-                                 " missing newlines or malformed?")
-
-    def update_start(self, start):
-        """Update the start time for the next contiguous interval.
-        Call this before starting to insert data for a new interval,
-        for example, after .finalize()"""
-        self._interval_start = start
-
-    def update_end(self, end):
-        """Update the end time for the current contiguous interval.
-        Call this before .finalize()"""
-        self._interval_end = end
-
-    def finalize(self):
-        """Stop filling the current contiguous interval.
-        All outstanding data will be sent, and the interval end
-        time of the interval will be taken from the 'end' argument
-        used when initializing this class, or the most recent
-        value passed to update_end(), or the last timestamp plus
-        a small epsilon value if no other endpoint was provided.
-
-        If more data is inserted after a finalize(), it will become
-        part of a new interval and there may be a gap left in-between."""
-        self._send_block(final=True)
-
-    def send(self):
-        """Send any data that we might have buffered up.  Does not affect
-        any other treatment of timestamps or endpoints."""
-        self._send_block(final=False)
-
-    def _get_first_noncomment(self, block):
-        """Return the (start, end) indices of the first full line in
-        block that isn't a comment, or raise IndexError if
-        there isn't one."""
-        start = 0
-        while True:
-            end = block.find(b'\n', start)
-            if end < 0:
-                raise IndexError
-            if block[start] != b'#'[0]:
-                return (start, (end + 1))
-            start = end + 1
-
-    def _get_last_noncomment(self, block):
-        """Return the (start, end) indices of the last full line in
-        block[:length] that isn't a comment, or raise IndexError if
-        there isn't one."""
-        end = block.rfind(b'\n')
-        if end <= 0:
-            raise IndexError
-        while True:
-            start = block.rfind(b'\n', 0, end)
-            if block[start + 1] != b'#'[0]:
-                return ((start + 1), end)
-            if start == -1:
-                raise IndexError
-            end = start
-
-    def _send_block(self, final=False):
-        """Send data currently in the block.  The data sent will
-        consist of full lines only, so some might be left over."""
-        # Build the full string to send
-        block = b"".join(self._block_data)
-
-        start_ts = self._interval_start
-        if start_ts is None:
-            # Pull start from the first line
-            try:
-                (spos, epos) = self._get_first_noncomment(block)
-                start_ts = extract_timestamp(block[spos:epos])
-            except (ValueError, IndexError):
-                pass  # no timestamp is OK, if we have no data
-
-        if final:
-            # For a final block, it must end in a newline, and the
-            # ending timestamp is either the user-provided end,
-            # or the timestamp of the last line plus epsilon.
-            end_ts = self._interval_end
-            try:
-                if block[-1] != b'\n'[0]:
-                    raise ValueError("final block didn't end with a newline")
-                if end_ts is None:
-                    (spos, epos) = self._get_last_noncomment(block)
-                    end_ts = extract_timestamp(block[spos:epos])
-                    end_ts += nilmdb.utils.time.epsilon
-            except (ValueError, IndexError):
-                pass  # no timestamp is OK, if we have no data
-            self._block_data = []
-            self._block_len = 0
-
-            # Next block is completely fresh
-            self._interval_start = None
-            self._interval_end = None
-        else:
-            # An intermediate block, e.g. "line1\nline2\nline3\nline4"
-            # We need to save "line3\nline4" for the next block, and
-            # use the timestamp from "line3" as the ending timestamp
-            # for this one.
-            try:
-                (spos, epos) = self._get_last_noncomment(block)
-                end_ts = extract_timestamp(block[spos:epos])
-            except (ValueError, IndexError):
-                # If we found no timestamp, give up; we could send this
-                # block later when we have more data.
-                return
-            if spos == 0:
-                # Not enough data to send an intermediate block
-                return
-            if self._interval_end is not None and end_ts > self._interval_end:
-                # User gave us bad endpoints; send it anyway, and let
-                # the server complain so that the error is the same
-                # as if we hadn't done this chunking.
-                end_ts = self._interval_end
-            self._block_data = [block[spos:]]
-            self._block_len = (epos - spos)
-            block = block[:spos]
-
-            # Next block continues where this one ended
-            self._interval_start = end_ts
-
-        # Double check endpoints
-        if (start_ts is None or end_ts is None) or (start_ts == end_ts):
-            # If the block has no non-comment lines, it's OK
-            try:
-                self._get_first_noncomment(block)
-            except IndexError:
-                return
-            raise ClientError("have data to send, but no start/end times")
-
-        # Send it
-        self.last_response = self._client.stream_insert_block(
-            self._path, block, start_ts, end_ts, binary=False)
-
-        return

nilmdb/client/errors.py

@@ -1,41 +0,0 @@
-"""HTTP client errors"""
-
-from nilmdb.utils.printf import sprintf
-
-
-class Error(Exception):
-    """Base exception for both ClientError and ServerError responses"""
-    def __init__(self,
-                 status="Unspecified error",
-                 message=None,
-                 url=None,
-                 traceback=None):
-        super().__init__(status)
-        self.status = status     # e.g. "400 Bad Request"
-        self.message = message   # textual message from the server
-        self.url = url           # URL we were requesting
-        self.traceback = traceback  # server traceback, if available
-
-    def _format_error(self, show_url):
-        s = sprintf("[%s]", self.status)
-        if self.message:
-            s += sprintf(" %s", self.message)
-        if show_url and self.url:
-            s += sprintf(" (%s)", self.url)
-        if self.traceback:
-            s += sprintf("\nServer traceback:\n%s", self.traceback)
-        return s
-
-    def __str__(self):
-        return self._format_error(show_url=False)
-
-    def __repr__(self):
-        return self._format_error(show_url=True)
-
-
-class ClientError(Error):
-    pass
-
-
-class ServerError(Error):
-    pass

nilmdb/client/httpclient.py

@@ -1,189 +0,0 @@
-"""HTTP client library"""
-
-import json
-import urllib.parse
-import requests
-
-from nilmdb.client.errors import ClientError, ServerError, Error
-
-
-class HTTPClient():
-    """Class to manage and perform HTTP requests from the client"""
-    def __init__(self, baseurl="", post_json=False, verify_ssl=True):
-        """If baseurl is supplied, all other functions that take
-        a URL can be given a relative URL instead."""
-        # Verify / clean up URL
-        reparsed = urllib.parse.urlparse(baseurl).geturl()
-        if '://' not in reparsed:
-            reparsed = urllib.parse.urlparse("http://" + baseurl).geturl()
-        self.baseurl = reparsed.rstrip('/') + '/'
-
-        # Note whether we want SSL verification
-        self.verify_ssl = verify_ssl
-
-        # Saved response, so that tests can verify a few things.
-        self._last_response = {}
-
-        # Whether to send application/json POST bodies (versus
-        # x-www-form-urlencoded)
-        self.post_json = post_json
-
-    def _handle_error(self, url, code, body):
-        # Default variables for exception.  We use the entire body as
-        # the default message, in case we can't extract it from a JSON
-        # response.
-        args = {
-            "url": url,
-            "status": str(code),
-            "message": body,
-            "traceback": None
-        }
-        try:
-            # Fill with server-provided data if we can
-            jsonerror = json.loads(body)
-            args["status"] = jsonerror["status"]
-            args["message"] = jsonerror["message"]
-            args["traceback"] = jsonerror["traceback"]
-        except Exception:
-            pass
-        if 400 <= code <= 499:
-            raise ClientError(**args)
-        else:
-            if 500 <= code <= 599:
-                if args["message"] is None:
-                    args["message"] = ("(no message; try disabling "
-                                       "response.stream option in "
-                                       "nilmdb.server for better debugging)")
-                raise ServerError(**args)
-            else:
-                raise Error(**args)
-
-    def close(self):
-        pass
-
-    def _do_req(self, method, url, query_data, body_data, stream, headers):
-        url = urllib.parse.urljoin(self.baseurl, url)
-        try:
-            # Create a new session, ensure we send "Connection: close",
-            # and explicitly close connection after the transfer.
-            # This is to avoid HTTP/1.1 persistent connections
-            # (keepalive), because they have fundamental race
-            # conditions when there are delays between requests:
-            # a new request may be sent at the same instant that the
-            # server decides to timeout the connection.
-            session = requests.Session()
-            if headers is None:
-                headers = {}
-            headers["Connection"] = "close"
-            response = session.request(method, url,
-                                       params=query_data,
-                                       data=body_data,
-                                       stream=stream,
-                                       headers=headers,
-                                       verify=self.verify_ssl)
-
-            # Close the connection.  If it's a generator (stream =
-            # True), the requests library shouldn't actually close the
-            # HTTP connection until all data has been read from the
-            # response.
-            session.close()
-        except requests.RequestException as e:
-            raise ServerError(status="502 Error", url=url,
-                              message=str(e))
-        if response.status_code != 200:
-            self._handle_error(url, response.status_code, response.content)
-        self._last_response = response
-        if response.headers["content-type"] in ("application/json",
-                                                "application/x-json-stream"):
-            return (response, True)
-        else:
-            return (response, False)
-
-    # Normal versions that return data directly
-    def _req(self, method, url, query=None, body=None, headers=None):
-        """
-        Make a request and return the body data as a string or parsed
-        JSON object, or raise an error if it contained an error.
-        """
-        (response, isjson) = self._do_req(method, url, query, body,
-                                          stream=False, headers=headers)
-        if isjson:
-            return json.loads(response.content)
-        return response.text
-
-    def get(self, url, params=None):
-        """Simple GET (parameters in URL)"""
-        return self._req("GET", url, params, None)
-
-    def post(self, url, params=None):
-        """Simple POST (parameters in body)"""
-        if self.post_json:
-            return self._req("POST", url, None,
-                             json.dumps(params),
-                             {'Content-type': 'application/json'})
-        else:
-            return self._req("POST", url, None, params)
-
-    def put(self, url, data, params=None,
-            content_type="application/octet-stream"):
-        """Simple PUT (parameters in URL, data in body)"""
-        h = {'Content-type': content_type}
-        return self._req("PUT", url, query=params, body=data, headers=h)
-
-    # Generator versions that return data one line at a time.
-    def _req_gen(self, method, url, query=None, body=None,
-                 headers=None, binary=False):
-        """
-        Make a request and return a generator that gives back strings
-        or JSON decoded lines of the body data, or raise an error if
-        it contained an eror.
-        """
-        (response, isjson) = self._do_req(method, url, query, body,
-                                          stream=True, headers=headers)
-
-        # Like the iter_lines function in Requests, but only splits on
-        # the specified line ending.
-        def lines(source, ending):
-            pending = None
-            for chunk in source:
-                if pending is not None:
-                    chunk = pending + chunk
-                tmp = chunk.split(ending)
-                lines = tmp[:-1]
-                if chunk.endswith(ending):
-                    pending = None
-                else:
-                    pending = tmp[-1]
-                for line in lines:
-                    yield line
-            if pending is not None:
-                yield pending
-
-        # Yield the chunks or lines as requested
-        if binary:
-            for chunk in response.iter_content(chunk_size=65536):
-                yield chunk
-        elif isjson:
-            for line in lines(response.iter_content(chunk_size=1),
-                              ending=b'\r\n'):
-                yield json.loads(line)
-        else:
-            for line in lines(response.iter_content(chunk_size=65536),
-                              ending=b'\n'):
-                yield line
-
-    def get_gen(self, url, params=None, binary=False):
-        """Simple GET (parameters in URL) returning a generator"""
-        return self._req_gen("GET", url, params, binary=binary)
-
-    def post_gen(self, url, params=None):
-        """Simple POST (parameters in body) returning a generator"""
-        if self.post_json:
-            return self._req_gen("POST", url, None,
-                                 json.dumps(params),
-                                 {'Content-type': 'application/json'})
-        else:
-            return self._req_gen("POST", url, None, params)
-
-    # Not much use for a POST or PUT generator, since they don't
-    # return much data.

nilmdb/client/numpyclient.py

@@ -1,263 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Provide a NumpyClient class that is based on normal Client, but has
-additional methods for extracting and inserting data via Numpy arrays."""
-
-import contextlib
-
-import numpy
-
-import nilmdb.utils
-import nilmdb.client.client
-import nilmdb.client.httpclient
-from nilmdb.client.errors import ClientError
-
-
-def layout_to_dtype(layout):
-    ltype = layout.split('_')[0]
-    lcount = int(layout.split('_')[1])
-    if ltype.startswith('int'):
-        atype = '<i' + str(int(ltype[3:]) // 8)
-    elif ltype.startswith('uint'):
-        atype = '<u' + str(int(ltype[4:]) // 8)
-    elif ltype.startswith('float'):
-        atype = '<f' + str(int(ltype[5:]) // 8)
-    else:
-        raise ValueError("bad layout")
-    if lcount == 1:
-        dtype = [('timestamp', '<i8'), ('data', atype)]
-    else:
-        dtype = [('timestamp', '<i8'), ('data', atype, lcount)]
-    return numpy.dtype(dtype)
-
-
-class NumpyClient(nilmdb.client.client.Client):
-    """Subclass of nilmdb.client.Client that adds additional methods for
-    extracting and inserting data via Numpy arrays."""
-
-    def _get_dtype(self, path, layout):
-        if layout is None:
-            streams = self.stream_list(path)
-            if len(streams) != 1:
-                raise ClientError("can't get layout for path: " + path)
-            layout = streams[0][1]
-        return layout_to_dtype(layout)
-
-    def stream_extract_numpy(self, path, start=None, end=None,
-                             layout=None, maxrows=100000,
-                             structured=False):
-        """
-        Extract data from a stream.  Returns a generator that yields
-        Numpy arrays of up to 'maxrows' of data each.
-
-        If 'layout' is None, it is read using stream_info.
-
-        If 'structured' is False, all data is converted to float64
-        and returned in a flat 2D array.  Otherwise, data is returned
-        as a structured dtype in a 1D array.
-        """
-        dtype = self._get_dtype(path, layout)
-
-        def to_numpy(data):
-            a = numpy.frombuffer(data, dtype)
-            if structured:
-                return a
-            return numpy.c_[a['timestamp'], a['data']]
-
-        chunks = []
-        total_len = 0
-        maxsize = dtype.itemsize * maxrows
-        for data in self.stream_extract(path, start, end, binary=True):
-            # Add this block of binary data
-            chunks.append(data)
-            total_len += len(data)
-
-            # See if we have enough to make the requested Numpy array
-            while total_len >= maxsize:
-                assembled = b"".join(chunks)
-                total_len -= maxsize
-                chunks = [assembled[maxsize:]]
-                block = assembled[:maxsize]
-                yield to_numpy(block)
-
-        if total_len:
-            yield to_numpy(b"".join(chunks))
-
-    @contextlib.contextmanager
-    def stream_insert_numpy_context(self, path, start=None, end=None,
-                                    layout=None):
-        """Return a context manager that allows data to be efficiently
-        inserted into a stream in a piecewise manner.  Data is
-        provided as Numpy arrays, and is aggregated and sent to the
-        server in larger or smaller chunks as necessary.  Data format
-        must match the database layout for the given path.
-
-        For more details, see help for
-        nilmdb.client.numpyclient.StreamInserterNumpy
-
-        If 'layout' is not None, use it as the layout rather than
-        querying the database.
-        """
-        dtype = self._get_dtype(path, layout)
-        ctx = StreamInserterNumpy(self, path, start, end, dtype)
-        yield ctx
-        ctx.finalize()
-        ctx.destroy()
-
-    def stream_insert_numpy(self, path, data, start=None, end=None,
-                            layout=None):
-        """Insert data into a stream.  data should be a Numpy array
-        which will be passed through stream_insert_numpy_context to
-        break it into chunks etc.  See the help for that function
-        for details."""
-        with self.stream_insert_numpy_context(path, start, end, layout) as ctx:
-            if isinstance(data, numpy.ndarray):
-                ctx.insert(data)
-            else:
-                for chunk in data:
-                    ctx.insert(chunk)
-        return ctx.last_response
-
-
-class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
-    """Object returned by stream_insert_numpy_context() that manages
-    the insertion of rows of data into a particular path.
-
-    See help for nilmdb.client.client.StreamInserter for details.
-    The only difference is that, instead of ASCII formatted data,
-    this context manager can take Numpy arrays, which are either
-    structured (1D with complex dtype) or flat (2D with simple dtype).
-    """
-
-    # Soft limit of how many bytes to send per HTTP request.
-    _max_data = 2 * 1024 * 1024
-
-    def __init__(self, client, path, start, end, dtype):
-        """
-        'client' is the client object.  'path' is the database path
-        to insert to.  'start' and 'end' are used for the first
-        contiguous interval and may be None.  'dtype' is the Numpy
-        dtype for this stream.
-        """
-        super(StreamInserterNumpy, self).__init__(client, path, start, end)
-        self._dtype = dtype
-
-        # Max rows to send at once
-        self._max_rows = self._max_data // self._dtype.itemsize
-
-        # List of the current arrays we're building up to send
-        self._block_arrays = []
-        self._block_rows = 0
-
-    def insert(self, array):
-        """Insert Numpy data, which must match the layout type."""
-        if not isinstance(array, numpy.ndarray):
-            array = numpy.array(array)
-        if array.ndim == 1:
-            # Already a structured array; just verify the type
-            if array.dtype != self._dtype:
-                raise ValueError("wrong dtype for 1D (structured) array")
-        elif array.ndim == 2:
-            # Convert to structured array
-            sarray = numpy.zeros(array.shape[0], dtype=self._dtype)
-            try:
-                sarray['timestamp'] = array[:, 0]
-                # Need the squeeze in case sarray['data'] is 1 dimensional
-                sarray['data'] = numpy.squeeze(array[:, 1:])
-            except (IndexError, ValueError):
-                raise ValueError("wrong number of fields for this data type")
-            array = sarray
-        else:
-            raise ValueError("wrong number of dimensions in array")
-
-        length = len(array)
-        maxrows = self._max_rows
-
-        if length == 0:
-            return
-        if length > maxrows:
-            # This is more than twice what we wanted to send, so split
-            # it up.  This is a bit inefficient, but the user really
-            # shouldn't be providing this much data at once.
-            for cut in range(0, length, maxrows):
-                self.insert(array[cut:(cut + maxrows)])
-            return
-
-        # Add this array to our list
-        self._block_arrays.append(array)
-        self._block_rows += length
-
-        # Send if it's too long
-        if self._block_rows >= maxrows:
-            self._send_block(final=False)
-
-    def _send_block(self, final=False):
-        """Send the data current stored up.  One row might be left
-        over if we need its timestamp saved."""
-
-        # Build the full array to send
-        if self._block_rows == 0:
-            array = numpy.zeros(0, dtype=self._dtype)
-        else:
-            array = numpy.hstack(self._block_arrays)
-
-        # Get starting timestamp
-        start_ts = self._interval_start
-        if start_ts is None:
-            # Pull start from the first row
-            try:
-                start_ts = array['timestamp'][0]
-            except IndexError:
-                pass  # no timestamp is OK, if we have no data
-
-        # Get ending timestamp
-        if final:
-            # For a final block, the timestamp is either the
-            # user-provided end, or the timestamp of the last line
-            # plus epsilon.
-            end_ts = self._interval_end
-            if end_ts is None:
-                try:
-                    end_ts = array['timestamp'][-1]
-                    end_ts += nilmdb.utils.time.epsilon
-                except IndexError:
-                    pass  # no timestamp is OK, if we have no data
-            self._block_arrays = []
-            self._block_rows = 0
-
-            # Next block is completely fresh
-            self._interval_start = None
-            self._interval_end = None
-        else:
-            # An intermediate block.  We need to save the last row
-            # for the next block, and use its timestamp as the ending
-            # timestamp for this one.
-            if len(array) < 2:
-                # Not enough data to send an intermediate block
-                return
-            end_ts = array['timestamp'][-1]
-            if self._interval_end is not None and end_ts > self._interval_end:
-                # User gave us bad endpoints; send it anyway, and let
-                # the server complain so that the error is the same
-                # as if we hadn't done this chunking.
-                end_ts = self._interval_end
-            self._block_arrays = [array[-1:]]
-            self._block_rows = 1
-            array = array[:-1]
-
-            # Next block continues where this one ended
-            self._interval_start = end_ts
-
-        # If we have no endpoints, or equal endpoints, it's OK as long
-        # as there's no data to send
-        if (start_ts is None or end_ts is None) or (start_ts == end_ts):
-            if not array:
-                return
-            raise ClientError("have data to send, but invalid start/end times")
-
-        # Send it
-        data = array.tostring()
-        self.last_response = self._client.stream_insert_block(
-            self._path, data, start_ts, end_ts, binary=True)
-
-        return

nilmdb/cmdline/__init__.py

@@ -1,3 +0,0 @@
-"""nilmdb.cmdline"""
-
-from nilmdb.cmdline.cmdline import Cmdline

nilmdb/cmdline/cmdline.py

@@ -1,173 +0,0 @@
-"""Command line client functionality"""
-
-import os
-import sys
-import signal
-import argparse
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-import nilmdb.client
-from nilmdb.utils.printf import fprintf, sprintf
-import nilmdb.utils.time
-
-import argcomplete
-import datetime_tz
-
-# Valid subcommands.  Defined in separate files just to break
-# things up -- they're still called with Cmdline as self.
-subcommands = ["help", "info", "create", "rename", "list", "intervals",
-               "metadata", "insert", "extract", "remove", "destroy"]
-
-# Import the subcommand modules
-subcmd_mods = {}
-for cmd in subcommands:
-    subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist=[cmd])
-
-
-class JimArgumentParser(argparse.ArgumentParser):
-    def parse_args(self, args=None, namespace=None):
-        # Look for --version anywhere and change it to just "nilmtool
-        # --version".  This makes "nilmtool cmd --version" work, which
-        # is needed by help2man.
-        if "--version" in (args or sys.argv[1:]):
-            args = ["--version"]
-        return argparse.ArgumentParser.parse_args(self, args, namespace)
-
-    def error(self, message):
-        self.print_usage(sys.stderr)
-        self.exit(2, sprintf("error: %s\n", message))
-
-
-class Complete():
-    # Completion helpers, for using argcomplete (see
-    # extras/nilmtool-bash-completion.sh)
-    def escape(self, s):
-        quote_chars = ["\\", "\"", "'", " "]
-        for char in quote_chars:
-            s = s.replace(char, "\\" + char)
-        return s
-
-    def none(self, prefix, parsed_args, **kwargs):
-        return []
-    rate = none
-    time = none
-    url = none
-
-    def path(self, prefix, parsed_args, **kwargs):
-        client = nilmdb.client.Client(parsed_args.url)
-        return (self.escape(s[0])
-                for s in client.stream_list()
-                if s[0].startswith(prefix))
-
-    def layout(self, prefix, parsed_args, **kwargs):
-        types = ["int8", "int16", "int32", "int64",
-                 "uint8", "uint16", "uint32", "uint64",
-                 "float32", "float64"]
-        layouts = []
-        for i in range(1, 10):
-            layouts.extend([(t + "_" + str(i)) for t in types])
-        return (lay for lay in layouts if lay.startswith(prefix))
-
-    def meta_key(self, prefix, parsed_args, **kwargs):
-        return (kv.split('=')[0] for kv
-                in self.meta_keyval(prefix, parsed_args, **kwargs))
-
-    def meta_keyval(self, prefix, parsed_args, **kwargs):
-        client = nilmdb.client.Client(parsed_args.url)
-        path = parsed_args.path
-        if not path:
-            return []
-        results = []
-        for (k, v) in client.stream_get_metadata(path).items():
-            kv = self.escape(k + '=' + v)
-            if kv.startswith(prefix):
-                results.append(kv)
-        return results
-
-
-class Cmdline():
-
-    def __init__(self, argv=None):
-        self.argv = argv or sys.argv[1:]
-        self.client = None
-        self.def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
-        self.subcmd = {}
-        self.complete = Complete()
-        self.complete_output_stream = None  # overridden by test suite
-
-    def arg_time(self, toparse):
-        """Parse a time string argument"""
-        try:
-            return nilmdb.utils.time.parse_time(toparse)
-        except ValueError as e:
-            raise argparse.ArgumentTypeError(sprintf("%s \"%s\"",
-                                                     str(e), toparse))
-
-    # Set up the parser
-    def parser_setup(self):
-        self.parser = JimArgumentParser(add_help=False,
-                                        formatter_class=def_form)
-
-        group = self.parser.add_argument_group("General options")
-        group.add_argument("-h", "--help", action='help',
-                           help='show this help message and exit')
-        group.add_argument("-v", "--version", action="version",
-                           version=nilmdb.__version__)
-
-        group = self.parser.add_argument_group("Server")
-        group.add_argument("-u", "--url", action="store",
-                           default=self.def_url,
-                           help="NilmDB server URL (default: %(default)s)"
-                           ).completer = self.complete.url
-
-        sub = self.parser.add_subparsers(
-            title="Commands", dest="command", required=True,
-            description="Use 'help command' or 'command --help' for more "
-            "details on a particular command.")
-
-        # Set up subcommands (defined in separate files)
-        for cmd in subcommands:
-            self.subcmd[cmd] = subcmd_mods[cmd].setup(self, sub)
-
-    def die(self, formatstr, *args):
-        fprintf(sys.stderr, formatstr + "\n", *args)
-        if self.client:
-            self.client.close()
-        sys.exit(-1)
-
-    def run(self):
-        # Set SIGPIPE to its default handler -- we don't need Python
-        # to catch it for us.
-        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-
-        # Clear cached timezone, so that we can pick up timezone changes
-        # while running this from the test suite.
-        datetime_tz._localtz = None
-
-        # Run parser
-        self.parser_setup()
-        argcomplete.autocomplete(self.parser, exit_method=sys.exit,
-                                 output_stream=self.complete_output_stream)
-        self.args = self.parser.parse_args(self.argv)
-
-        # Run arg verify handler if there is one
-        if "verify" in self.args:
-            self.args.verify(self)
-
-        self.client = nilmdb.client.Client(self.args.url)
-
-        # Make a test connection to make sure things work,
-        # unless the particular command requests that we don't.
-        if "no_test_connect" not in self.args:
-            try:
-                self.client.version()
-            except nilmdb.client.Error as e:
-                self.die("error connecting to server: %s", str(e))
-
-        # Now dispatch client request to appropriate function.  Parser
-        # should have ensured that we don't have any unknown commands
-        # here.
-        retval = self.args.handler(self) or 0
-
-        self.client.close()
-        sys.exit(retval)

nilmdb/cmdline/create.py

@@ -1,38 +0,0 @@
-from argparse import RawDescriptionHelpFormatter as raw_form
-
-import nilmdb.client
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("create", help="Create a new stream",
-                         formatter_class=raw_form,
-                         description="""
-Create a new empty stream at the specified path and with the specified
-layout type.
-
-Layout types are of the format: type_count
-
-  'type' is a data type like 'float32', 'float64', 'uint16', 'int32', etc.
-
-  'count' is the number of columns of this type.
-
-  For example, 'float32_8' means the data for this stream has 8 columns of
-  32-bit floating point values.
-""")
-    cmd.set_defaults(handler=cmd_create)
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("path",
-                       help="Path (in database) of new stream, e.g. /foo/bar",
-                       ).completer = self.complete.path
-    group.add_argument("layout",
-                       help="Layout type for new stream, e.g. float32_8",
-                       ).completer = self.complete.layout
-    return cmd
-
-
-def cmd_create(self):
-    """Create new stream"""
-    try:
-        self.client.stream_create(self.args.path, self.args.layout)
-    except nilmdb.client.ClientError as e:
-        self.die("error creating stream: %s", str(e))

nilmdb/cmdline/destroy.py

@@ -1,52 +0,0 @@
-import fnmatch
-
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-from nilmdb.utils.printf import printf
-import nilmdb.client
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("destroy", help="Delete a stream and all data",
-                         formatter_class=def_form,
-                         description="""
-                         Destroy the stream at the specified path.
-                         The stream must be empty.  All metadata
-                         related to the stream is permanently deleted.
-
-                         Wildcards and multiple paths are supported.
-                         """)
-    cmd.set_defaults(handler=cmd_destroy)
-    group = cmd.add_argument_group("Options")
-    group.add_argument("-R", "--remove", action="store_true",
-                       help="Remove all data before destroying stream")
-    group.add_argument("-q", "--quiet", action="store_true",
-                       help="Don't display names when destroying "
-                       "multiple paths")
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("path", nargs='+',
-                       help="Path of the stream to delete, e.g. /foo/bar/*",
-                       ).completer = self.complete.path
-    return cmd
-
-
-def cmd_destroy(self):
-    """Destroy stream"""
-    streams = [s[0] for s in self.client.stream_list()]
-    paths = []
-    for path in self.args.path:
-        new = fnmatch.filter(streams, path)
-        if not new:
-            self.die("error: no stream matched path: %s", path)
-        paths.extend(new)
-
-    for path in paths:
-        if not self.args.quiet and len(paths) > 1:
-            printf("Destroying %s\n", path)
-
-        try:
-            if self.args.remove:
-                self.client.stream_remove(path)
-            self.client.stream_destroy(path)
-        except nilmdb.client.ClientError as e:
-            self.die("error destroying stream: %s", str(e))

nilmdb/cmdline/extract.py

@@ -1,96 +0,0 @@
-import sys
-
-from nilmdb.utils.printf import printf
-import nilmdb.client
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("extract", help="Extract data",
-                         description="""
-                         Extract data from a stream.
-                         """)
-    cmd.set_defaults(verify=cmd_extract_verify,
-                     handler=cmd_extract)
-
-    group = cmd.add_argument_group("Data selection")
-    group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar",
-                       ).completer = self.complete.path
-    group.add_argument("-s", "--start", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)",
-                       ).completer = self.complete.time
-    group.add_argument("-e", "--end", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)",
-                       ).completer = self.complete.time
-
-    group = cmd.add_argument_group("Output format")
-    group.add_argument("-B", "--binary", action="store_true",
-                       help="Raw binary output")
-    group.add_argument("-b", "--bare", action="store_true",
-                       help="Exclude timestamps from output lines")
-    group.add_argument("-a", "--annotate", action="store_true",
-                       help="Include comments with some information "
-                       "about the stream")
-    group.add_argument("-m", "--markup", action="store_true",
-                       help="Include comments with interval starts and ends")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps in annotated information")
-    group.add_argument("-c", "--count", action="store_true",
-                       help="Just output a count of matched data points")
-    return cmd
-
-
-def cmd_extract_verify(self):
-    if self.args.start > self.args.end:
-        self.parser.error("start is after end")
-
-    if self.args.binary:
-        if (self.args.bare or self.args.annotate or self.args.markup or
-                self.args.timestamp_raw or self.args.count):
-            self.parser.error("--binary cannot be combined with other options")
-
-
-def cmd_extract(self):
-    streams = self.client.stream_list(self.args.path)
-    if len(streams) != 1:
-        self.die("error getting stream info for path %s", self.args.path)
-    layout = streams[0][1]
-
-    if self.args.timestamp_raw:
-        time_string = nilmdb.utils.time.timestamp_to_string
-    else:
-        time_string = nilmdb.utils.time.timestamp_to_human
-
-    if self.args.annotate:
-        printf("# path: %s\n", self.args.path)
-        printf("# layout: %s\n", layout)
-        printf("# start: %s\n", time_string(self.args.start))
-        printf("# end: %s\n", time_string(self.args.end))
-
-    printed = False
-    if self.args.binary:
-        printer = sys.stdout.buffer.write
-    else:
-        printer = lambda x: print(x.decode('utf-8'))
-    bare = self.args.bare
-    count = self.args.count
-    for dataline in self.client.stream_extract(self.args.path,
-                                               self.args.start,
-                                               self.args.end,
-                                               self.args.count,
-                                               self.args.markup,
-                                               self.args.binary):
-        if bare and not count:
-            # Strip timestamp (first element).  Doesn't make sense
-            # if we are only returning a count.
-            dataline = b' '.join(dataline.split(b' ')[1:])
-        printer(dataline)
-        printed = True
-    if not printed:
-        if self.args.annotate:
-            printf("# no data\n")
-        return 2
-
-    return 0

nilmdb/cmdline/help.py

@@ -1,25 +0,0 @@
-import argparse
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("help", help="Show detailed help for a command",
-                         description="""
-                         Show help for a command. 'help command' is
-                         the same as 'command --help'.
-                         """)
-    cmd.set_defaults(handler=cmd_help)
-    cmd.set_defaults(no_test_connect=True)
-    cmd.add_argument("command", nargs="?",
-                     help="Command to get help about")
-    cmd.add_argument("rest", nargs=argparse.REMAINDER,
-                     help=argparse.SUPPRESS)
-    return cmd
-
-
-def cmd_help(self):
-    if self.args.command in self.subcmd:
-        self.subcmd[self.args.command].print_help()
-    else:
-        self.parser.print_help()
-
-    return

nilmdb/cmdline/info.py

@@ -1,30 +0,0 @@
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-import nilmdb.client
-from nilmdb.utils.printf import printf
-from nilmdb.utils import human_size
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("info", help="Server information",
-                         formatter_class=def_form,
-                         description="""
-                         List information about the server, like
-                         version.
-                         """)
-    cmd.set_defaults(handler=cmd_info)
-    return cmd
-
-
-def cmd_info(self):
-    """Print info about the server"""
-    printf("Client version: %s\n", nilmdb.__version__)
-    printf("Server version: %s\n", self.client.version())
-    printf("Server URL: %s\n", self.client.geturl())
-    dbinfo = self.client.dbinfo()
-    printf("Server database path: %s\n", dbinfo["path"])
-    for (desc, field) in [("used by NilmDB", "size"),
-                          ("used by other", "other"),
-                          ("reserved", "reserved"),
-                          ("free", "free")]:
-        printf("Server disk space %s: %s\n", desc, human_size(dbinfo[field]))

nilmdb/cmdline/insert.py

@@ -1,135 +0,0 @@
-import sys
-
-from nilmdb.utils.printf import printf
-import nilmdb.client
-import nilmdb.utils.timestamper as timestamper
-import nilmdb.utils.time
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("insert", help="Insert data",
-                         description="""
-                         Insert data into a stream.
-                         """)
-    cmd.set_defaults(verify=cmd_insert_verify,
-                     handler=cmd_insert)
-    cmd.add_argument("-q", "--quiet", action='store_true',
-                     help='suppress unnecessary messages')
-
-    group = cmd.add_argument_group("Timestamping",
-                                   description="""
-                                   To add timestamps, specify the
-                                   arguments --timestamp and --rate,
-                                   and provide a starting time.
-                                   """)
-
-    group.add_argument("-t", "--timestamp", action="store_true",
-                       help="Add timestamps to each line")
-    group.add_argument("-r", "--rate", type=float,
-                       help="Data rate, in Hz",
-                       ).completer = self.complete.rate
-
-    group = cmd.add_argument_group("Start time",
-                                   description="""
-                                   Start time may be manually
-                                   specified with --start, or guessed
-                                   from the filenames using
-                                   --filename.  Set the TZ environment
-                                   variable to change the default
-                                   timezone.""")
-
-    exc = group.add_mutually_exclusive_group()
-    exc.add_argument("-s", "--start",
-                     metavar="TIME", type=self.arg_time,
-                     help="Starting timestamp (free-form)",
-                     ).completer = self.complete.time
-    exc.add_argument("-f", "--filename", action="store_true",
-                     help="Use filename to determine start time")
-
-    group = cmd.add_argument_group("End time",
-                                   description="""
-                                   End time for the overall stream.
-                                   (required when not using --timestamp).
-                                   Set the TZ environment
-                                   variable to change the default
-                                   timezone.""")
-    group.add_argument("-e", "--end",
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form)",
-                       ).completer = self.complete.time
-
-    group = cmd.add_argument_group("Required parameters")
-    group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar",
-                       ).completer = self.complete.path
-    group.add_argument("file", nargs='?', default='-',
-                       help="File to insert (default: - (stdin))")
-    return cmd
-
-
-def cmd_insert_verify(self):
-    if self.args.timestamp:
-        if not self.args.rate:
-            self.die("error: --rate is needed, but was not specified")
-        if not self.args.filename and self.args.start is None:
-            self.die("error: need --start or --filename "
-                     "when adding timestamps")
-    else:
-        if self.args.start is None or self.args.end is None:
-            self.die("error: when not adding timestamps, --start and "
-                     "--end are required")
-
-
-def cmd_insert(self):
-    # Find requested stream
-    streams = self.client.stream_list(self.args.path)
-    if len(streams) != 1:
-        self.die("error getting stream info for path %s", self.args.path)
-
-    arg = self.args
-
-    try:
-        filename = arg.file
-        if filename == '-':
-            infile = sys.stdin.buffer
-        else:
-            try:
-                infile = open(filename, "rb")
-            except IOError:
-                self.die("error opening input file %s", filename)
-
-        if arg.start is None:
-            try:
-                arg.start = nilmdb.utils.time.parse_time(filename)
-            except ValueError:
-                self.die("error extracting start time from filename '%s'",
-                         filename)
-
-        if arg.timestamp:
-            data = timestamper.TimestamperRate(infile, arg.start, arg.rate)
-        else:
-            data = iter(lambda: infile.read(1048576), b'')
-
-        # Print info
-        if not arg.quiet:
-            printf(" Input file: %s\n", filename)
-            printf(" Start time: %s\n",
-                   nilmdb.utils.time.timestamp_to_human(arg.start))
-            if arg.end:
-                printf("   End time: %s\n",
-                       nilmdb.utils.time.timestamp_to_human(arg.end))
-            if arg.timestamp:
-                printf("Timestamper: %s\n", str(data))
-
-        # Insert the data
-        self.client.stream_insert(arg.path, data, arg.start, arg.end)
-
-    except nilmdb.client.Error as e:
-        # TODO: It would be nice to be able to offer better errors
-        # here, particularly in the case of overlap, which just shows
-        # ugly bracketed ranges of 16-digit numbers and a mangled URL.
-        # Need to consider adding something like e.prettyprint()
-        # that is smarter about the contents of the error.
-        self.die("error inserting data: %s", str(e))
-
-    return

nilmdb/cmdline/intervals.py

@@ -1,76 +0,0 @@
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-from nilmdb.utils.printf import printf
-import nilmdb.utils.time
-from nilmdb.utils.interval import Interval
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("intervals", help="List intervals",
-                         formatter_class=def_form,
-                         description="""
-                         List intervals in a stream, similar to
-                         'list --detail path'.
-
-                         If '--diff diffpath' is provided, only
-                         interval ranges that are present in 'path'
-                         and not present in 'diffpath' are printed.
-                         """)
-    cmd.set_defaults(verify=cmd_intervals_verify,
-                     handler=cmd_intervals)
-
-    group = cmd.add_argument_group("Stream selection")
-    group.add_argument("path", metavar="PATH",
-                       help="List intervals for this path",
-                       ).completer = self.complete.path
-    group.add_argument("-d", "--diff", metavar="PATH",
-                       help="Subtract intervals from this path",
-                       ).completer = self.complete.path
-
-    group = cmd.add_argument_group("Interval details")
-    group.add_argument("-s", "--start",
-                       metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp for intervals "
-                       "(free-form, inclusive)",
-                       ).completer = self.complete.time
-    group.add_argument("-e", "--end",
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp for intervals "
-                       "(free-form, noninclusive)",
-                       ).completer = self.complete.time
-
-    group = cmd.add_argument_group("Misc options")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps when printing times")
-    group.add_argument("-o", "--optimize", action="store_true",
-                       help="Optimize (merge adjacent) intervals")
-
-    return cmd
-
-
-def cmd_intervals_verify(self):
-    if self.args.start is not None and self.args.end is not None:
-        if self.args.start >= self.args.end:
-            self.parser.error("start must precede end")
-
-
-def cmd_intervals(self):
-    """List intervals in a stream"""
-    if self.args.timestamp_raw:
-        time_string = nilmdb.utils.time.timestamp_to_string
-    else:
-        time_string = nilmdb.utils.time.timestamp_to_human
-
-    try:
-        intervals = (Interval(start, end) for (start, end) in
-                     self.client.stream_intervals(self.args.path,
-                                                  self.args.start,
-                                                  self.args.end,
-                                                  self.args.diff))
-        if self.args.optimize:
-            intervals = nilmdb.utils.interval.optimize(intervals)
-        for i in intervals:
-            printf("[ %s -> %s ]\n", time_string(i.start), time_string(i.end))
-
-    except nilmdb.client.ClientError as e:
-        self.die("error listing intervals: %s", str(e))

nilmdb/cmdline/list.py

@@ -1,105 +0,0 @@
-import fnmatch
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-from nilmdb.utils.printf import printf
-import nilmdb.utils.time
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("list", help="List streams",
-                         formatter_class=def_form,
-                         description="""
-                         List streams available in the database,
-                         optionally filtering by path.  Wildcards
-                         are accepted; non-matching paths or wildcards
-                         are ignored.
-                         """)
-    cmd.set_defaults(verify=cmd_list_verify,
-                     handler=cmd_list)
-
-    group = cmd.add_argument_group("Stream filtering")
-    group.add_argument("path", metavar="PATH", default=["*"], nargs='*',
-                       ).completer = self.complete.path
-
-    group = cmd.add_argument_group("Interval info")
-    group.add_argument("-E", "--ext", action="store_true",
-                       help="Show extended stream info, like interval "
-                       "extents and row count")
-
-    group = cmd.add_argument_group("Interval details")
-    group.add_argument("-d", "--detail", action="store_true",
-                       help="Show available data time intervals")
-    group.add_argument("-s", "--start",
-                       metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp for intervals "
-                       "(free-form, inclusive)",
-                       ).completer = self.complete.time
-    group.add_argument("-e", "--end",
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp for intervals "
-                       "(free-form, noninclusive)",
-                       ).completer = self.complete.time
-
-    group = cmd.add_argument_group("Misc options")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps when printing times")
-    group.add_argument("-l", "--layout", action="store_true",
-                       help="Show layout type next to path name")
-    group.add_argument("-n", "--no-decim", action="store_true",
-                       help="Skip paths containing \"~decim-\"")
-
-    return cmd
-
-
-def cmd_list_verify(self):
-    if self.args.start is not None and self.args.end is not None:
-        if self.args.start >= self.args.end:
-            self.parser.error("start must precede end")
-
-    if self.args.start is not None or self.args.end is not None:
-        if not self.args.detail:
-            self.parser.error("--start and --end only make sense "
-                              "with --detail")
-
-
-def cmd_list(self):
-    """List available streams"""
-    streams = self.client.stream_list(extended=True)
-
-    if self.args.timestamp_raw:
-        time_string = nilmdb.utils.time.timestamp_to_string
-    else:
-        time_string = nilmdb.utils.time.timestamp_to_human
-
-    for argpath in self.args.path:
-        for stream in streams:
-            (path, layout, int_min, int_max, rows, time) = stream[:6]
-            if not fnmatch.fnmatch(path, argpath):
-                continue
-            if self.args.no_decim and "~decim-" in path:
-                continue
-
-            if self.args.layout:
-                printf("%s %s\n", path, layout)
-            else:
-                printf("%s\n", path)
-
-            if self.args.ext:
-                if int_min is None or int_max is None:
-                    printf("  interval extents: (no data)\n")
-                else:
-                    printf("  interval extents: %s -> %s\n",
-                           time_string(int_min), time_string(int_max))
-                printf("        total data: %d rows, %.6f seconds\n",
-                       rows or 0,
-                       nilmdb.utils.time.timestamp_to_seconds(time or 0))
-
-            if self.args.detail:
-                printed = False
-                for (start, end) in self.client.stream_intervals(
-                        path, self.args.start, self.args.end):
-                    printf("  [ %s -> %s ]\n",
-                           time_string(start), time_string(end))
-                    printed = True
-                if not printed:
-                    printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -1,90 +0,0 @@
-from nilmdb.utils.printf import printf
-import nilmdb
-import nilmdb.client
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("metadata", help="Get or set stream metadata",
-                         description="""
-                         Get or set key=value metadata associated with
-                         a stream.
-                         """,
-                         usage="%(prog)s path [-g [key ...] | "
-                         "-s key=value [...] | -u key=value [...]] | "
-                         "-d [key ...]")
-    cmd.set_defaults(handler=cmd_metadata)
-
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("path",
-                       help="Path of stream, e.g. /foo/bar",
-                       ).completer = self.complete.path
-
-    group = cmd.add_argument_group("Actions")
-    exc = group.add_mutually_exclusive_group()
-    exc.add_argument("-g", "--get", nargs="*", metavar="key",
-                     help="Get metadata for specified keys (default all)",
-                     ).completer = self.complete.meta_key
-    exc.add_argument("-s", "--set", nargs="+", metavar="key=value",
-                     help="Replace all metadata with provided "
-                     "key=value pairs",
-                     ).completer = self.complete.meta_keyval
-    exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
-                     help="Update metadata using provided "
-                     "key=value pairs",
-                     ).completer = self.complete.meta_keyval
-    exc.add_argument("-d", "--delete", nargs="*", metavar="key",
-                     help="Delete metadata for specified keys (default all)",
-                     ).completer = self.complete.meta_key
-    return cmd
-
-
-def cmd_metadata(self):
-    """Manipulate metadata"""
-    if self.args.set is not None or self.args.update is not None:
-        # Either set, or update
-        if self.args.set is not None:
-            keyvals = self.args.set
-            handler = self.client.stream_set_metadata
-        else:
-            keyvals = self.args.update
-            handler = self.client.stream_update_metadata
-
-        # Extract key=value pairs
-        data = {}
-        for keyval in keyvals:
-            kv = keyval.split('=')
-            if len(kv) != 2 or kv[0] == "":
-                self.die("error parsing key=value argument '%s'", keyval)
-            data[kv[0]] = kv[1]
-
-        # Make the call
-        try:
-            handler(self.args.path, data)
-        except nilmdb.client.ClientError as e:
-            self.die("error setting/updating metadata: %s", str(e))
-    elif self.args.delete is not None:
-        # Delete (by setting values to empty strings)
-        keys = None
-        if self.args.delete:
-            keys = list(self.args.delete)
-        try:
-            data = self.client.stream_get_metadata(self.args.path, keys)
-            for key in data:
-                data[key] = ""
-            self.client.stream_update_metadata(self.args.path, data)
-        except nilmdb.client.ClientError as e:
-            self.die("error deleting metadata: %s", str(e))
-    else:
-        # Get (or unspecified)
-        keys = None
-        if self.args.get:
-            keys = list(self.args.get)
-        try:
-            data = self.client.stream_get_metadata(self.args.path, keys)
-        except nilmdb.client.ClientError as e:
-            self.die("error getting metadata: %s", str(e))
-        for key, value in sorted(data.items()):
-            # Print nonexistant keys as having empty value
-            if value is None:
-                value = ""
-            printf("%s=%s\n", key, value)

nilmdb/cmdline/remove.py

@@ -1,59 +0,0 @@
-import fnmatch
-
-from nilmdb.utils.printf import printf
-import nilmdb.client
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("remove", help="Remove data",
-                         description="""
-                         Remove all data from a specified time range within a
-                         stream.  If multiple streams or wildcards are
-                         provided, the same time range is removed from all
-                         streams.
-                         """)
-    cmd.set_defaults(handler=cmd_remove)
-
-    group = cmd.add_argument_group("Data selection")
-    group.add_argument("path", nargs='+',
-                       help="Path of stream, e.g. /foo/bar/*",
-                       ).completer = self.complete.path
-    group.add_argument("-s", "--start", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)",
-                       ).completer = self.complete.time
-    group.add_argument("-e", "--end", required=True,
-                       metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)",
-                       ).completer = self.complete.time
-
-    group = cmd.add_argument_group("Output format")
-    group.add_argument("-q", "--quiet", action="store_true",
-                       help="Don't display names when removing "
-                       "from multiple paths")
-    group.add_argument("-c", "--count", action="store_true",
-                       help="Output number of data points removed")
-    return cmd
-
-
-def cmd_remove(self):
-    streams = [s[0] for s in self.client.stream_list()]
-    paths = []
-    for path in self.args.path:
-        new = fnmatch.filter(streams, path)
-        if not new:
-            self.die("error: no stream matched path: %s", path)
-        paths.extend(new)
-
-    try:
-        for path in paths:
-            if not self.args.quiet and len(paths) > 1:
-                printf("Removing from %s\n", path)
-            count = self.client.stream_remove(path,
-                                              self.args.start, self.args.end)
-            if self.args.count:
-                printf("%d\n", count)
-    except nilmdb.client.ClientError as e:
-        self.die("error removing data: %s", str(e))
-
-    return 0

nilmdb/cmdline/rename.py

@@ -1,32 +0,0 @@
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-import nilmdb.client
-
-
-def setup(self, sub):
-    cmd = sub.add_parser("rename", help="Rename a stream",
-                         formatter_class=def_form,
-                         description="""
-                         Rename a stream.
-
-                         Only the stream's path is renamed; no
-                         metadata is changed.
-                         """)
-    cmd.set_defaults(handler=cmd_rename)
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("oldpath",
-                       help="Old path, e.g. /foo/old",
-                       ).completer = self.complete.path
-    group.add_argument("newpath",
-                       help="New path, e.g. /foo/bar/new",
-                       ).completer = self.complete.path
-
-    return cmd
-
-
-def cmd_rename(self):
-    """Rename a stream"""
-    try:
-        self.client.stream_rename(self.args.oldpath, self.args.newpath)
-    except nilmdb.client.ClientError as e:
-        self.die("error renaming stream: %s", str(e))

nilmdb/fileinterval.py

@@ -0,0 +1,37 @@
+"""FileInterval
+
+An Interval that is backed with file data storage"""
+
+from nilmdb.interval import Interval, IntervalSet, IntervalError
+from datetime import datetime
+import bisect
+
+class FileInterval(Interval):
+    """Represents an interval of time and its corresponding data"""
+
+    def __init__(self, start, end, 
+                 filename, 
+                 start_offset = None, end_offset = None):
+        self.start = start
+        self.end = end
+        self.filename = filename
+        if start_offset is None:
+            start_offset = 0
+        self.start_offset = start_offset
+        if end_offset is None:
+            f = open(filename, 'rb')
+            f.seek(0, os.SEEK_END)
+            end_offset = f.tell()
+        self.end_offset = end_offset
+
+    def __setattr__(self, name, value):
+        pass
+
+    def subset(self, start, end):
+        """Return a new Interval that is a subset of this one"""
+        # TODO: Any magic regarding file/offset/length mapping for subsets
+        if (start < self.start or end > self.end):
+            raise IntervalError("not a subset")
+        return FileInterval(start, end)            
+
+    

nilmdb/fsck/__init__.py

@@ -1,3 +0,0 @@
-"""nilmdb.fsck"""
-
-from nilmdb.fsck.fsck import Fsck

nilmdb/fsck/fsck.py

@@ -1,610 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Check database consistency, with some ability to fix problems.
-This should be able to fix cases where a database gets corrupted due
-to unexpected system shutdown, and detect other cases that may cause
-NilmDB to return errors when trying to manipulate the database."""
-
-import nilmdb.utils
-import nilmdb.server
-import nilmdb.client.numpyclient
-from nilmdb.utils.interval import IntervalError
-from nilmdb.server.interval import Interval, IntervalSet
-from nilmdb.utils.printf import printf, fprintf, sprintf
-
-from collections import defaultdict
-import sqlite3
-import os
-import sys
-import progressbar
-import re
-import shutil
-import pickle
-import numpy
-
-
-class FsckError(Exception):
-    def __init__(self, msg="", *args):
-        if args:
-            msg = sprintf(msg, *args)
-        Exception.__init__(self, msg)
-
-
-class FixableFsckError(FsckError):
-    def __init__(self, msg=""):
-        FsckError.__init__(self, f'{msg}\nThis may be fixable with "--fix".')
-
-
-class RetryFsck(FsckError):
-    pass
-
-
-class FsckFormatError(FsckError):
-    pass
-
-
-def log(format, *args):
-    printf(format, *args)
-
-
-def err(format, *args):
-    fprintf(sys.stderr, format, *args)
-
-
-# Decorator that retries a function if it returns a specific value
-def retry_if_raised(exc, message=None, max_retries=1000):
-    def f1(func):
-        def f2(*args, **kwargs):
-            for n in range(max_retries):
-                try:
-                    return func(*args, **kwargs)
-                except exc:
-                    if message:
-                        log(f"{message} ({n+1})\n\n")
-            raise Exception("Max number of retries (%d) exceeded; giving up" %
-                            max_retries)
-        return f2
-    return f1
-
-
-class Progress(object):
-    def __init__(self, maxval):
-        if maxval == 0:
-            maxval = 1
-        self.bar = progressbar.ProgressBar(
-            maxval=maxval,
-            widgets=[progressbar.Percentage(), ' ',
-                     progressbar.Bar(), ' ',
-                     progressbar.ETA()])
-        self.bar.term_width = self.bar.term_width or 75
-
-    def __enter__(self):
-        self.bar.start()
-        self.last_update = 0
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        if exc_type is None:
-            self.bar.finish()
-        else:
-            printf("\n")
-
-    def update(self, val):
-        self.bar.update(val)
-
-
-class Fsck(object):
-    def __init__(self, path, fix=False):
-        self.basepath = path
-        self.sqlpath = os.path.join(path, "data.sql")
-        self.bulkpath = os.path.join(path, "data")
-        self.bulklock = os.path.join(path, "data.lock")
-        self.fix = fix
-
-    ### Main checks
-
-    @retry_if_raised(RetryFsck, "Something was fixed: restarting fsck")
-    def check(self, skip_data=False):
-        self.bulk = None
-        self.sql = None
-        try:
-            self.check_paths()
-            self.check_sql()
-            self.check_streams()
-            self.check_intervals()
-            if skip_data:
-                log("skipped data check\n")
-            else:
-                self.check_data()
-        finally:
-            if self.bulk:
-                self.bulk.close()
-            if self.sql:  # pragma: no cover
-                # (coverage doesn't handle finally clauses correctly;
-                # both branches here are tested)
-                self.sql.commit()
-                self.sql.close()
-        log("ok\n")
-
-    ### Check basic path structure
-
-    def check_paths(self):
-        log("checking paths\n")
-        if self.bulk:
-            self.bulk.close()
-        if not os.path.isfile(self.sqlpath):
-            raise FsckError("SQL database missing (%s)", self.sqlpath)
-        if not os.path.isdir(self.bulkpath):
-            raise FsckError("Bulk data directory missing (%s)", self.bulkpath)
-        with open(self.bulklock, "w") as lockfile:
-            if not nilmdb.utils.lock.exclusive_lock(lockfile):
-                raise FsckError('Database already locked by another process\n'
-                                'Make sure all other processes that might be '
-                                'using the database are stopped.\n'
-                                'Restarting apache will cause it to unlock '
-                                'the db until a request is received.')
-            # unlocked immediately
-        self.bulk = nilmdb.server.bulkdata.BulkData(self.basepath)
-
-    ### Check SQL database health
-
-    def check_sql(self):
-        log("checking sqlite database\n")
-
-        self.sql = sqlite3.connect(self.sqlpath)
-        with self.sql:
-            cur = self.sql.cursor()
-            ver = cur.execute("PRAGMA user_version").fetchone()[0]
-            good = max(nilmdb.server.nilmdb._sql_schema_updates.keys())
-            if ver != good:
-                raise FsckError("database version %d too old, should be %d",
-                                ver, good)
-            self.stream_path = {}
-            self.stream_layout = {}
-            log("  loading paths\n")
-            result = cur.execute("SELECT id, path, layout FROM streams")
-            for r in result:
-                if r[0] in self.stream_path:
-                    raise FsckError("duplicated ID %d in stream IDs", r[0])
-                self.stream_path[r[0]] = r[1]
-                self.stream_layout[r[0]] = r[2]
-
-            log("  loading intervals\n")
-            self.stream_interval = defaultdict(list)
-            result = cur.execute("SELECT stream_id, start_time, end_time, "
-                                 "start_pos, end_pos FROM ranges "
-                                 "ORDER BY start_time")
-            for r in result:
-                if r[0] not in self.stream_path:
-                    raise FsckError("interval ID %d not in streams", r[0])
-                self.stream_interval[r[0]].append((r[1], r[2], r[3], r[4]))
-
-            log("  loading metadata\n")
-            self.stream_meta = defaultdict(dict)
-            result = cur.execute("SELECT stream_id, key, value FROM metadata")
-            for r in result:
-                if r[0] not in self.stream_path:
-                    raise FsckError("metadata ID %d not in streams", r[0])
-                if r[1] in self.stream_meta[r[0]]:
-                    raise FsckError(
-                        "duplicate metadata key '%s' for stream %d",
-                        r[1], r[0])
-                self.stream_meta[r[0]][r[1]] = r[2]
-
-    ### Check streams and basic interval overlap
-
-    def check_streams(self):
-        ids = list(self.stream_path.keys())
-        log("checking %s streams\n", "{:,d}".format(len(ids)))
-        with Progress(len(ids)) as pbar:
-            for i, sid in enumerate(ids):
-                pbar.update(i)
-                path = self.stream_path[sid]
-
-                # unique path, valid layout
-                if list(self.stream_path.values()).count(path) != 1:
-                    raise FsckError("duplicated path %s", path)
-                layout = self.stream_layout[sid].split('_')[0]
-                if layout not in ('int8', 'int16', 'int32', 'int64',
-                                  'uint8', 'uint16', 'uint32', 'uint64',
-                                  'float32', 'float64'):
-                    raise FsckError("bad layout %s for %s", layout, path)
-                count = int(self.stream_layout[sid].split('_')[1])
-                if count < 1 or count > 1024:
-                    raise FsckError("bad count %d for %s", count, path)
-
-                # must exist in bulkdata
-                bulk = self.bulkpath + path
-                bulk = bulk.encode('utf-8')
-                if not os.path.isdir(bulk):
-                    raise FsckError("%s: missing bulkdata dir", path)
-                if not nilmdb.server.bulkdata.Table.exists(bulk):
-                    raise FsckError("%s: bad bulkdata table", path)
-
-                # intervals don't overlap.  Abuse IntervalSet to check
-                # for intervals in file positions, too.
-                timeiset = IntervalSet()
-                posiset = IntervalSet()
-                for (stime, etime, spos, epos) in self.stream_interval[sid]:
-                    new = Interval(stime, etime)
-                    try:
-                        timeiset += new
-                    except IntervalError:
-                        raise FsckError("%s: overlap in intervals:\n"
-                                        "set: %s\nnew: %s",
-                                        path, str(timeiset), str(new))
-                    if spos != epos:
-                        new = Interval(spos, epos)
-                        try:
-                            posiset += new
-                        except IntervalError:
-                            self.fix_row_overlap(sid, path, posiset, new)
-
-                try:
-                    # Check bulkdata
-                    self.check_bulkdata(sid, path, bulk)
-
-                    # Check that we can open bulkdata
-                    tab = nilmdb.server.bulkdata.Table(bulk)
-                except FsckFormatError:
-                    # If there are no files except _format, try deleting
-                    # the entire stream; this may remove metadata, but
-                    # it's probably unimportant.
-                    files = list(os.listdir(bulk))
-                    if len(files) > 1:
-                        raise FsckFormatError(f"{path}: can't load _format, "
-                                              f"but data is also present")
-
-                    # Since the stream was empty, just remove it
-                    self.fix_remove_stream(sid, path, bulk,
-                                           "empty, with corrupted format file")
-                except FsckError as e:
-                    raise e
-                except Exception as e:  # pragma: no cover
-                    # No coverage because this is an unknown/unexpected error
-                    raise FsckError("%s: can't open bulkdata: %s",
-                                    path, str(e))
-                tab.close()
-
-    def fix_row_overlap(self, sid, path, existing, new):
-        # If the file rows (spos, epos) overlap in the interval table,
-        # and the overlapping ranges look like this:
-        #    A --------- C
-        #           B -------- D
-        # Then we can try changing the first interval to go from
-        # A to B instead.
-        msg = (f"{path}: overlap in file offsets:\n"
-               f"existing ranges: {existing}\n"
-               f"overlapping interval: {new}")
-        if not self.fix:
-            raise FixableFsckError(msg)
-        err(f"\n{msg}\nSeeing if we can truncate one of them...\n")
-
-        # See if there'e exactly one interval that overlaps the
-        # conflicting one in the right way
-        match = None
-        for intv in self.stream_interval[sid]:
-            (stime, etime, spos, epos) = intv
-            if spos < new.start and epos > new.start:
-                if match:
-                    err(f"no, more than one interval matched:\n"
-                        f"{intv}\n{match}\n")
-                    raise FsckError(f"{path}: unfixable overlap")
-                match = intv
-        if match is None:
-            err("no intervals overlapped in the right way\n")
-            raise FsckError(f"{path}: unfixable overlap")
-
-        # Truncate the file position
-        err(f"truncating {match}\n")
-        with self.sql:
-            cur = self.sql.cursor()
-            cur.execute("UPDATE ranges SET end_pos=? "
-                        "WHERE stream_id=? AND start_time=? AND "
-                        "end_time=? AND start_pos=? AND end_pos=?",
-                        (new.start, sid, *match))
-            if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
-                raise FsckError("failed to fix SQL database")
-        raise RetryFsck
-
-    ### Check that bulkdata is good enough to be opened
-
-    @retry_if_raised(RetryFsck)
-    def check_bulkdata(self, sid, path, bulk):
-        try:
-            with open(os.path.join(bulk, b"_format"), "rb") as f:
-                fmt = pickle.load(f)
-        except Exception as e:
-            raise FsckFormatError(f"{path}: can't load _format file ({e})")
-
-        if fmt["version"] != 3:
-            raise FsckFormatError("%s: bad or unsupported bulkdata version %d",
-                                  path, fmt["version"])
-        rows_per_file = int(fmt["rows_per_file"])
-        if rows_per_file < 1:
-            raise FsckFormatError(f"{path}: bad rows_per_file {rows_per_file}")
-        files_per_dir = int(fmt["files_per_dir"])
-        if files_per_dir < 1:
-            raise FsckFormatError(f"{path}: bad files_per_dir {files_per_dir}")
-        layout = fmt["layout"]
-        if layout != self.stream_layout[sid]:
-            raise FsckFormatError("%s: layout mismatch %s != %s", path,
-                                  layout, self.stream_layout[sid])
-
-        # Every file should have a size that's the multiple of the row size
-        rkt = nilmdb.server.rocket.Rocket(layout, None)
-        row_size = rkt.binary_size
-        rkt.close()
-
-        # Find all directories
-        regex = re.compile(b"^[0-9a-f]{4,}$")
-        subdirs = sorted(filter(regex.search, os.listdir(bulk)),
-                         key=lambda x: int(x, 16), reverse=True)
-        for subdir in subdirs:
-            # Find all files in that dir
-            subpath = os.path.join(bulk, subdir)
-            files = list(filter(regex.search, os.listdir(subpath)))
-            if not files:
-                self.fix_empty_subdir(subpath)
-
-            # Verify that their size is a multiple of the row size
-            for filename in files:
-                filepath = os.path.join(subpath, filename)
-                offset = os.path.getsize(filepath)
-                if offset % row_size:
-                    self.fix_bad_filesize(path, filepath, offset, row_size)
-
-    def fix_empty_subdir(self, subpath):
-        msg = sprintf("bulkdata path %s is missing data files", subpath)
-        if not self.fix:
-            raise FixableFsckError(msg)
-        # Try to fix it by just deleting whatever is present,
-        # as long as it's only ".removed" files.
-        err("\n%s\n", msg)
-        for fn in os.listdir(subpath):
-            if not fn.endswith(b".removed"):
-                raise FsckError("can't fix automatically: please manually "
-                                "remove the file '%s' and try again",
-                                os.path.join(subpath, fn).decode(
-                                    'utf-8', errors='backslashreplace'))
-        # Remove the whole thing
-        err("Removing empty subpath\n")
-        shutil.rmtree(subpath)
-        raise RetryFsck
-
-    def fix_bad_filesize(self, path, filepath, offset, row_size):
-        extra = offset % row_size
-        msg = sprintf("%s: size of file %s (%d) is not a multiple" +
-                      " of row size (%d): %d extra bytes present",
-                      path, filepath, offset, row_size, extra)
-        if not self.fix:
-            raise FixableFsckError(msg)
-        # Try to fix it by just truncating the file
-        err("\n%s\n", msg)
-        newsize = offset - extra
-        err("Truncating file to %d bytes and retrying\n", newsize)
-        with open(filepath, "r+b") as f:
-            f.truncate(newsize)
-            raise RetryFsck
-
-    def fix_remove_stream(self, sid, path, bulk, reason):
-        msg = f"stream {path} is corrupted: {reason}"
-        if not self.fix:
-            raise FixableFsckError(msg)
-        # Remove the stream from disk and the database
-        err(f"\n{msg}\n")
-        err(f"Removing stream {path} from disk and database\n")
-        shutil.rmtree(bulk)
-        with self.sql:
-            cur = self.sql.cursor()
-            cur.execute("DELETE FROM streams WHERE id=?",
-                        (sid,))
-            if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
-                raise FsckError("failed to remove stream")
-            cur.execute("DELETE FROM ranges WHERE stream_id=?", (sid,))
-            cur.execute("DELETE FROM metadata WHERE stream_id=?", (sid,))
-        raise RetryFsck
-
-    ### Check interval endpoints
-
-    def check_intervals(self):
-        total_ints = sum(len(x) for x in list(self.stream_interval.values()))
-        log("checking %s intervals\n", "{:,d}".format(total_ints))
-        done = 0
-        with Progress(total_ints) as pbar:
-            for sid in self.stream_interval:
-                try:
-                    bulk = self.bulkpath + self.stream_path[sid]
-                    bulk = bulk.encode('utf-8')
-                    tab = nilmdb.server.bulkdata.Table(bulk)
-
-                    def update(x):
-                        pbar.update(done + x)
-
-                    ints = self.stream_interval[sid]
-                    done += self.check_table_intervals(sid, ints, tab, update)
-                finally:
-                    tab.close()
-
-    def check_table_intervals(self, sid, ints, tab, update):
-        # look in the table to make sure we can pick out the interval's
-        # endpoints
-        path = self.stream_path[sid]  # noqa: F841 unused
-        tab.file_open.cache_remove_all()
-        for (i, intv) in enumerate(ints):
-            update(i)
-            (stime, etime, spos, epos) = intv
-            if spos == epos and spos >= 0 and spos <= tab.nrows:
-                continue
-            try:
-                srow = tab[spos]    # noqa: F841 unused
-                erow = tab[epos-1]  # noqa: F841 unused
-            except Exception as e:
-                self.fix_bad_interval(sid, intv, tab, str(e))
-
-        return len(ints)
-
-    def fix_bad_interval(self, sid, intv, tab, msg):
-        path = self.stream_path[sid]
-        msg = sprintf("%s: interval %s error accessing rows: %s",
-                      path, str(intv), str(msg))
-        if not self.fix:
-            raise FixableFsckError(msg)
-        err("\n%s\n", msg)
-
-        (stime, etime, spos, epos) = intv
-        # If it's just that the end pos is more than the number of rows
-        # in the table, lower end pos and truncate interval time too.
-        if spos < tab.nrows and epos >= tab.nrows:
-            err("end position is past endrows, but it can be truncated\n")
-            err("old end: time %d, pos %d\n", etime, epos)
-            new_epos = tab.nrows
-            new_etime = tab[new_epos-1] + 1
-            err("new end: time %d, pos %d\n", new_etime, new_epos)
-            if stime < new_etime:
-                # Change it in SQL
-                with self.sql:
-                    cur = self.sql.cursor()
-                    cur.execute("UPDATE ranges SET end_time=?, end_pos=? "
-                                "WHERE stream_id=? AND start_time=? AND "
-                                "end_time=? AND start_pos=? AND end_pos=?",
-                                (new_etime, new_epos, sid, stime, etime,
-                                 spos, epos))
-                    if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
-                        raise FsckError("failed to fix SQL database")
-                raise RetryFsck
-            err("actually it can't be truncated; times are bad too\n")
-
-        # Otherwise, the only hope is to delete the interval entirely.
-        err("*** Deleting the entire interval from SQL.\n")
-        err("This may leave stale data on disk.  To fix that, copy all "
-            "data from this stream to a new stream using nilm-copy, then\n")
-        err("remove all data from and destroy %s.\n", path)
-        with self.sql:
-            cur = self.sql.cursor()
-            cur.execute("DELETE FROM ranges WHERE "
-                        "stream_id=? AND start_time=? AND "
-                        "end_time=? AND start_pos=? AND end_pos=?",
-                        (sid, stime, etime, spos, epos))
-            if cur.rowcount != 1:  # pragma: no cover (shouldn't fail)
-                raise FsckError("failed to remove interval")
-        raise RetryFsck
-
-    ### Check data in each interval
-
-    def check_data(self):
-        total_rows = sum(sum((y[3] - y[2]) for y in x)
-                         for x in list(self.stream_interval.values()))
-        log("checking %s rows of data\n", "{:,d}".format(total_rows))
-        done = 0
-        with Progress(total_rows) as pbar:
-            for sid in self.stream_interval:
-                try:
-                    bulk = self.bulkpath + self.stream_path[sid]
-                    bulk = bulk.encode('utf-8')
-                    tab = nilmdb.server.bulkdata.Table(bulk)
-
-                    def update(x):
-                        pbar.update(done + x)
-
-                    ints = self.stream_interval[sid]
-                    done += self.check_table_data(sid, ints, tab, update)
-                finally:
-                    tab.close()
-
-    def check_table_data(self, sid, ints, tab, update):
-        # Pull out all of the interval's data and verify that it's
-        # monotonic.
-        maxrows = getattr(self, 'maxrows_override', 100000)
-        path = self.stream_path[sid]
-        layout = self.stream_layout[sid]
-        dtype = nilmdb.client.numpyclient.layout_to_dtype(layout)
-        tab.file_open.cache_remove_all()
-        done = 0
-        for intv in ints:
-            last_ts = None
-            (stime, etime, spos, epos) = intv
-
-            # Break interval into maxrows-sized chunks
-            next_start = spos
-            while next_start < epos:
-                start = next_start
-                stop = min(start + maxrows, epos)
-                count = stop - start
-                next_start = stop
-
-                # Get raw data, convert to NumPy arary
-                try:
-                    raw = tab.get_data(start, stop, binary=True)
-                    data = numpy.frombuffer(raw, dtype)
-                except Exception as e:  # pragma: no cover
-                    # No coverage because it's hard to trigger this -- earlier
-                    # checks check the ranges, so this would probably be a real
-                    # disk error, malloc failure, etc.
-                    raise FsckError(
-                        "%s: failed to grab rows %d through %d: %s",
-                        path, start, stop, repr(e))
-
-                ts = data['timestamp']
-
-                # Verify that all timestamps are in range.
-                match = (ts < stime) | (ts >= etime)
-                if match.any():
-                    row = numpy.argmax(match)
-                    if ts[row] != 0:
-                        raise FsckError("%s: data timestamp %d at row %d "
-                                        "outside interval range [%d,%d)",
-                                        path, ts[row], row + start,
-                                        stime, etime)
-
-                    # Timestamp is zero and out of the expected range;
-                    # assume file ends with zeroed data and just truncate it.
-                    self.fix_table_by_truncating(
-                        path, tab, row + start,
-                        "data timestamp is out of range, and zero")
-
-                # Verify that timestamps are monotonic
-                match = numpy.diff(ts) <= 0
-                if match.any():
-                    row = numpy.argmax(match)
-                    if ts[row+1] != 0:
-                        raise FsckError(
-                            "%s: non-monotonic timestamp (%d -> %d) "
-                            "at row %d", path, ts[row], ts[row+1],
-                            row + start)
-
-                    # Timestamp is zero and non-monotonic;
-                    # assume file ends with zeroed data and just truncate it.
-                    self.fix_table_by_truncating(
-                        path, tab, row + start + 1,
-                        "data timestamp is non-monotonic, and zero")
-
-                first_ts = ts[0]
-                if last_ts is not None and first_ts <= last_ts:
-                    raise FsckError("%s: first interval timestamp %d is not "
-                                    "greater than the previous last interval "
-                                    "timestamp %d, at row %d",
-                                    path, first_ts, last_ts, start)
-                last_ts = ts[-1]
-
-                # The previous errors are fixable, by removing the
-                # offending intervals, or changing the data
-                # timestamps.  But these are probably unlikely errors,
-                # so it's not worth implementing that yet.
-
-                # Done
-                done += count
-                update(done)
-        return done
-
-    def fix_table_by_truncating(self, path, tab, row, reason):
-        # Simple fix for bad data: truncate the table at the given row.
-        # On retry, fix_bad_interval will correct the database and timestamps
-        # to account for this truncation.
-        msg = f"{path}: bad data in table, starting at row {row}: {reason}"
-        if not self.fix:
-            raise FixableFsckError(msg)
-        err(f"\n{msg}\nWill try truncating table\n")
-        (subdir, fname, offs, count) = tab._offset_from_row(row)
-        tab._remove_or_truncate_file(subdir, fname, offs)
-        raise RetryFsck

nilmdb/interval.py

@@ -0,0 +1,205 @@
+"""Interval and IntervalSet
+
+Represents an interval of time, and a sorted set of such intervals"""
+
+from datetime import datetime
+import bisect
+
+class IntervalError(Exception):
+    """Error due to interval overlap, etc"""
+    pass
+
+class Interval(object):
+    """Represents an interval of time"""
+
+    start = None
+    end = None
+
+    def __init__(self, start, end):
+        self.start = start
+        self.end = end
+
+    def __repr__(self):
+        return "Interval(" + repr(self.start) + ", " + repr(self.end) + ")"
+
+    def __str__(self):
+        return "[" + str(self.start) + " -> " + str(self.end) + "]"
+
+    def __setattr__(self, name, value):
+        """Set attribute"""
+        # TODO: If we need to manipulate file names, offsets, lengths, etc,
+        # based on start and end time changing, maybe this is the right spot?
+        # Or we could just disallow changing it here.
+        if not isinstance(value, datetime):
+            raise IntervalError("Must set datetime values")
+        self.__dict__[name] = value
+        if (type(self.start) is type(self.end)):
+            if (self.start > self.end):
+                raise IntervalError("Interval start must precede interval end")
+
+    def __cmp__(self, other):
+        """Compare two intervals.  If non-equal, order by start then end"""
+        if not isinstance(other, Interval):
+            raise TypeError("Can't compare to non-interval")
+        if (self.start == other.start):
+            if (self.end < other.end):
+                return -1
+            if (self.end > other.end):
+                return 1
+            return 0
+        if (self.start < other.start):
+            return -1
+        return 1
+                
+    def intersects(self, other):
+        """Return True if two Interval objects intersect"""
+        if (not isinstance(other, Interval)):
+            raise TypeError("need Interval for intersection test")
+        if (self.end <= other.start or
+            self.start >= other.end):
+            return False
+        else:
+            return True
+
+    def is_adjacent(self, other):
+        """Return True if two Intervals are adjacent (same end or start)"""
+        if (not isinstance(other, Interval)):
+            raise TypeError("need Interval for adjacency test")
+        if (self.end == other.start or
+            self.start == other.end):
+            return True
+        else:
+            return False
+
+    def subset(self, start, end):
+        """Return a new Interval that is a subset of this one"""
+        # TODO: Any magic regarding file/offset/length mapping for subsets
+        if (start < self.start or end > self.end):
+            raise IntervalError("not a subset")
+        return Interval(start, end)            
+
+class IntervalSet(object):
+    """A non-intersecting set of intervals
+
+    Kept sorted internally"""
+
+    def __init__(self, iterable=None):
+        self.data = []
+        if iterable is not None:
+            if isinstance(iterable, Interval):
+                iterable = [iterable]
+            self._add_intervals(iterable)
+
+    def __iter__(self):
+        return self.data.__iter__()
+
+    def __repr__(self):
+        return "IntervalSet(" + repr(list(self.data)) + ")"
+
+    def __cmp__(self, other):
+        # compare isn't supported, they don't really have an ordering
+        raise TypeError("can't compare IntervalSets with cmp()")
+
+    def __eq__(self, other):
+        """Test equality of two IntervalSets.
+
+        Treats adjacent Intervals as equivalent to one long interval,
+        so this function really tests whether the IntervalSets cover
+        the same spans of time."""
+        if not isinstance(other, IntervalSet):
+            return False
+        i = 0
+        j = 0
+        outside = True
+        try:
+            while True:
+                if (outside):
+                    # To match, we need to be finished this set
+                    if (i >= len(self) and j >= len(other)):
+                        return True
+                    # Or the starts need to match
+                    if (self[i].start != other[j].start):
+                        return False
+                    outside = False
+                else:
+                    # We can move on if the two interval ends match
+                    if (self[i].end == other[j].end):
+                        i += 1
+                        j += 1
+                        outside = True
+                    else:
+                        # Whichever ends first needs to be adjacent to the next
+                        if (self[i].end < other[j].end):
+                            if (not self[i].is_adjacent(self[i+1])):
+                                return False
+                            i += 1
+                        else:
+                            if (not other[j].is_adjacent(other[j+1])):
+                                return False
+                            j += 1
+        except IndexError:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, key):
+        return self.data.__getitem__(key)
+
+    def __iadd__(self, other):
+        """Inplace add -- modifies self
+
+        This throws an exception if the regions being added intersect."""
+        if isinstance(other, Interval):
+            other = [other]
+        self._add_intervals(other)
+        return self
+        
+    def __add__(self, other):
+        """Add -- returns a new object
+
+        This throws an exception if the regions being added intersect."""
+        new = IntervalSet(self)
+        new += IntervalSet(other)
+        return new
+
+    def __and__(self, other):
+        """Compute a new IntervalSet from the intersection of two others
+
+        Output intervals are built as subsets of the intervals in the
+        first argument (self)."""
+        # If we were given a set, intersect with each interval in that set
+        if isinstance(other, IntervalSet):
+            out = IntervalSet()
+            for interval in other.data:
+                out += self & interval
+            return out
+
+        if not isinstance(other, Interval):
+            raise TypeError("can't intersect with that type")
+
+        out = IntervalSet()
+        for this in self.data:
+            # If there's any overlap, add the overlapping region
+            if (this.end > other.start and this.start < other.end):
+                out += this.subset(max(this.start, other.start),
+                                   min(this.end, other.end))
+        return out
+    
+    def _add_intervals(self, iterable):
+        """Add each Interval from an interable to this set"""
+        for element in iter(iterable):
+            self._add_single_interval(element)
+
+    def _add_single_interval(self, interval):
+        """Add one Interval to this set"""
+        if (not isinstance(interval, Interval)):
+            raise TypeError("can only add Intervals")
+        for existing in self.data:
+            if existing.intersects(interval):
+                raise IntervalError("Tried to add overlapping interval "
+                                    "to this set")
+        bisect.insort(self.data, interval)

nilmdb/scripts/__init__.py

@@ -1 +0,0 @@
-# Command line scripts

nilmdb/scripts/nilmdb_fsck.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-import nilmdb.fsck
-import argparse
-
-
-def main():
-    """Main entry point for the 'nilmdb-fsck' command line script"""
-
-    parser = argparse.ArgumentParser(
-        description='Check database consistency',
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument("-v", "--version", action="version",
-                        version=nilmdb.__version__)
-    parser.add_argument("-f", "--fix", action="store_true",
-                        default=False, help='Fix errors when possible '
-                        '(which may involve removing data)')
-    parser.add_argument("-n", "--no-data", action="store_true",
-                        default=False, help='Skip the slow full-data check')
-    parser.add_argument('database', help='Database directory')
-    args = parser.parse_args()
-
-    nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data=args.no_data)
-
-
-if __name__ == "__main__":
-    main()

nilmdb/scripts/nilmdb_server.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import socket
-import argparse
-
-import cherrypy
-
-import nilmdb.server
-
-
-def main():
-    """Main entry point for the 'nilmdb-server' command line script"""
-
-    parser = argparse.ArgumentParser(
-        description='Run the NilmDB server',
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
-    parser.add_argument("-v", "--version", action="version",
-                        version=nilmdb.__version__)
-
-    group = parser.add_argument_group("Standard options")
-    group.add_argument('-a', '--address',
-                       help='Only listen on the given address',
-                       default='0.0.0.0')
-    group.add_argument('-p', '--port', help='Listen on the given port',
-                       type=int, default=12380)
-    group.add_argument('-d', '--database', help='Database directory',
-                       default="./db")
-    group.add_argument('-q', '--quiet', help='Silence output',
-                       action='store_true')
-    group.add_argument('-t', '--traceback',
-                       help='Provide tracebacks in client errors',
-                       action='store_true', default=False)
-
-    group = parser.add_argument_group("Debug options")
-    group.add_argument('-y', '--yappi', help='Run under yappi profiler and '
-                       'invoke interactive shell afterwards',
-                       action='store_true')
-
-    args = parser.parse_args()
-
-    # Create database object.  Needs to be serialized before passing
-    # to the Server.
-    db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database)
-
-    # Configure the server
-    if not args.quiet:
-        cherrypy._cpconfig.environments['embedded']['log.screen'] = True
-
-    server = nilmdb.server.Server(db,
-                                  host=args.address,
-                                  port=args.port,
-                                  force_traceback=args.traceback)
-
-    # Print info
-    if not args.quiet:
-        print("Version: %s" % nilmdb.__version__)
-        print("Database: %s" % (os.path.realpath(args.database)))
-        if args.address == '0.0.0.0' or args.address == '::':
-            host = socket.getfqdn()
-        else:
-            host = args.address
-        print("Server URL: http://%s:%d/" % (host, args.port))
-        print("----")
-
-    # Run it
-    try:
-        if args.yappi:
-            print("Running in yappi")
-            try:
-                import yappi
-                yappi.start()
-                server.start(blocking=True)
-            finally:
-                yappi.stop()
-                stats = yappi.get_func_stats()
-                stats.sort("ttot")
-                stats.print_all()
-                try:
-                    from IPython import embed
-                    embed(header="Use the `yappi` or `stats` object to "
-                          "explore further, `quit` to exit")
-                except ModuleNotFoundError:
-                    print("\nInstall ipython to explore further")
-        else:
-            server.start(blocking=True)
-    except nilmdb.server.serverutil.CherryPyExit:
-        print("Exiting due to CherryPy error", file=sys.stderr)
-        raise
-    finally:
-        if not args.quiet:
-            print("Closing database")
-            db.close()
-
-
-if __name__ == "__main__":
-    main()

nilmdb/scripts/nilmtool.py

@@ -1,12 +0,0 @@
-#!/usr/bin/env python3
-
-import nilmdb.cmdline
-
-
-def main():
-    """Main entry point for the 'nilmtool' command line script"""
-    nilmdb.cmdline.Cmdline().run()
-
-
-if __name__ == "__main__":
-    main()

nilmdb/server/__init__.py

@@ -1,9 +0,0 @@
-"""nilmdb.server"""
-
-# Set up pyximport to automatically rebuild Cython modules if needed.
-import pyximport
-pyximport.install(inplace=True, build_in_temp=False)
-
-from nilmdb.server.nilmdb import NilmDB
-from nilmdb.server.server import Server, wsgi_application
-from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

nilmdb/server/bulkdata.py

@@ -1,635 +0,0 @@
-# Fixed record size bulk data storage
-
-import os
-import re
-import sys
-import pickle
-import tempfile
-
-from nilmdb.utils.printf import sprintf
-from nilmdb.utils.time import timestamp_to_string
-import nilmdb.utils
-
-import nilmdb.utils.lock
-from . import rocket
-
-# Up to 256 open file descriptors at any given time.
-# These variables are global so they can be used in the decorator arguments.
-table_cache_size = 32
-fd_cache_size = 8
-
-
-@nilmdb.utils.must_close(wrap_verify=False)
-class BulkData():
-    def __init__(self, basepath, **kwargs):
-        if isinstance(basepath, str):
-            self.basepath = self._encode_filename(basepath)
-        else:
-            self.basepath = basepath
-        self.root = os.path.join(self.basepath, b"data")
-        self.lock = self.root + b".lock"
-        self.lockfile = None
-
-        # Tuneables
-        if "file_size" in kwargs and kwargs["file_size"] is not None:
-            self.file_size = kwargs["file_size"]
-        else:
-            # Default to approximately 128 MiB per file
-            self.file_size = 128 * 1024 * 1024
-
-        if "files_per_dir" in kwargs and kwargs["files_per_dir"] is not None:
-            self.files_per_dir = kwargs["files_per_dir"]
-        else:
-            # 32768 files per dir should work even on FAT32
-            self.files_per_dir = 32768
-
-        if "initial_nrows" in kwargs and kwargs["initial_nrows"] is not None:
-            self.initial_nrows = kwargs["initial_nrows"]
-        else:
-            # First row is 0
-            self.initial_nrows = 0
-
-        # Make root path
-        if not os.path.isdir(self.root):
-            os.mkdir(self.root)
-
-        # Create the lock
-        self.lockfile = open(self.lock, "w")
-        if not nilmdb.utils.lock.exclusive_lock(self.lockfile):
-            raise IOError('database at "' +
-                          self._decode_filename(self.basepath) +
-                          '" is already locked by another process')
-
-    def close(self):
-        self.getnode.cache_remove_all()
-        if self.lockfile:
-            nilmdb.utils.lock.exclusive_unlock(self.lockfile)
-            self.lockfile.close()
-            try:
-                os.unlink(self.lock)
-            except OSError:
-                pass
-            self.lockfile = None
-
-    def _encode_filename(self, path):
-        # Translate unicode strings to raw bytes, if needed.  We
-        # always manipulate paths internally as bytes.
-        return path.encode('utf-8')
-
-    def _decode_filename(self, path):
-        # Translate raw bytes to unicode strings, escaping if needed
-        return path.decode('utf-8', errors='backslashreplace')
-
-    def _create_check_ospath(self, ospath):
-        if ospath[-1:] == b'/':
-            raise ValueError("invalid path; should not end with a /")
-        if Table.exists(ospath):
-            raise ValueError("stream already exists at this path")
-        if os.path.isdir(ospath):
-            # Look for any files in subdirectories.  Fully empty subdirectories
-            # are OK; they might be there during a rename
-            for (root, dirs, files) in os.walk(ospath):
-                if files:
-                    raise ValueError(
-                        "non-empty subdirs of this path already exist")
-
-    def _create_parents(self, unicodepath):
-        """Verify the path name, and create parent directories if they
-        don't exist.  Returns a list of elements that got created."""
-        path = self._encode_filename(unicodepath)
-
-        if path[0:1] != b'/':
-            raise ValueError("paths must start with / ")
-        [group, node] = path.rsplit(b"/", 1)
-        if group == b'':
-            raise ValueError("invalid path; path must contain at least one "
-                             "folder")
-        if node == b'':
-            raise ValueError("invalid path; should not end with a /")
-        if not Table.valid_path(path):
-            raise ValueError("path name is invalid or contains reserved words")
-
-        # Create the table's base dir.  Note that we make a
-        # distinction here between NilmDB paths (always Unix style,
-        # split apart manually) and OS paths (built up with
-        # os.path.join)
-
-        # Make directories leading up to this one
-        elements = path.lstrip(b'/').split(b'/')
-        made_dirs = []
-        try:
-            # Make parent elements
-            for i in range(len(elements)):
-                ospath = os.path.join(self.root, *elements[0:i])
-                if Table.exists(ospath):
-                    raise ValueError("path is subdir of existing node")
-                if not os.path.isdir(ospath):
-                    os.mkdir(ospath)
-                    made_dirs.append(ospath)
-        except Exception:
-            # Remove paths that we created
-            for ospath in reversed(made_dirs):
-                os.rmdir(ospath)
-            raise
-
-        return elements
-
-    def create(self, unicodepath, layout_name):
-        """
-        unicodepath: path to the data (e.g. u'/newton/prep').
-        Paths must contain at least two elements, e.g.:
-           /newton/prep
-           /newton/raw
-           /newton/upstairs/prep
-           /newton/upstairs/raw
-
-        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
-        """
-        elements = self._create_parents(unicodepath)
-
-        # Make the final dir
-        ospath = os.path.join(self.root, *elements)
-        self._create_check_ospath(ospath)
-        os.mkdir(ospath)
-
-        try:
-            # Write format string to file
-            Table.create(ospath, layout_name, self.file_size,
-                         self.files_per_dir)
-
-            # Open and cache it
-            self.getnode(unicodepath)
-        except Exception:
-            exc_info = sys.exc_info()
-            try:
-                os.rmdir(ospath)
-            except OSError:
-                pass
-            raise exc_info[1].with_traceback(exc_info[2])
-
-        # Success
-        return
-
-    def _remove_leaves(self, unicodepath):
-        """Remove empty directories starting at the leaves of unicodepath"""
-        path = self._encode_filename(unicodepath)
-        elements = path.lstrip(b'/').split(b'/')
-        for i in reversed(list(range(len(elements)))):
-            ospath = os.path.join(self.root, *elements[0:i+1])
-            try:
-                os.rmdir(ospath)
-            except OSError:
-                pass
-
-    def rename(self, oldunicodepath, newunicodepath):
-        """Move entire tree from 'oldunicodepath' to
-        'newunicodepath'"""
-        oldpath = self._encode_filename(oldunicodepath)
-        newpath = self._encode_filename(newunicodepath)
-
-        # Get OS paths
-        oldelements = oldpath.lstrip(b'/').split(b'/')
-        oldospath = os.path.join(self.root, *oldelements)
-        newelements = newpath.lstrip(b'/').split(b'/')
-        newospath = os.path.join(self.root, *newelements)
-
-        # Basic checks
-        if oldospath == newospath:
-            raise ValueError("old and new paths are the same")
-
-        # Remove Table object at old path from cache
-        self.getnode.cache_remove(self, oldunicodepath)
-
-        # Move the table to a temporary location
-        tmpdir = tempfile.mkdtemp(prefix=b"rename-", dir=self.root)
-        tmppath = os.path.join(tmpdir, b"table")
-        os.rename(oldospath, tmppath)
-
-        try:
-            # Check destination path
-            self._create_check_ospath(newospath)
-
-            # Create parent dirs for new location
-            self._create_parents(newunicodepath)
-
-            # Move table into new location
-            os.rename(tmppath, newospath)
-        except Exception:
-            # On failure, move the table back to original path
-            os.rename(tmppath, oldospath)
-            os.rmdir(tmpdir)
-            raise
-
-        # Prune old dirs
-        self._remove_leaves(oldunicodepath)
-        os.rmdir(tmpdir)
-
-    def destroy(self, unicodepath):
-        """Fully remove all data at a particular path.  No way to undo
-        it!  The group/path structure is removed, too."""
-        path = self._encode_filename(unicodepath)
-
-        # Get OS path
-        elements = path.lstrip(b'/').split(b'/')
-        ospath = os.path.join(self.root, *elements)
-
-        # Remove Table object from cache
-        self.getnode.cache_remove(self, unicodepath)
-
-        # Remove the contents of the target directory
-        if not Table.exists(ospath):
-            raise ValueError("nothing at that path")
-        for (root, dirs, files) in os.walk(ospath, topdown=False):
-            for name in files:
-                os.remove(os.path.join(root, name))
-            for name in dirs:
-                os.rmdir(os.path.join(root, name))
-
-        # Remove leftover empty directories
-        self._remove_leaves(unicodepath)
-
-    # Cache open tables
-    @nilmdb.utils.lru_cache(size=table_cache_size,
-                            onremove=lambda x: x.close())
-    def getnode(self, unicodepath):
-        """Return a Table object corresponding to the given database
-        path, which must exist."""
-        path = self._encode_filename(unicodepath)
-        elements = path.lstrip(b'/').split(b'/')
-        ospath = os.path.join(self.root, *elements)
-        return Table(ospath, self.initial_nrows)
-
-
-@nilmdb.utils.must_close(wrap_verify=False)
-class Table():
-    """Tools to help access a single table (data at a specific OS path)."""
-    # See design.md for design details
-
-    # Class methods, to help keep format details in this class.
-    @classmethod
-    def valid_path(cls, root):
-        """Return True if a root path is a valid name"""
-        return b"_format" not in root.split(b"/")
-
-    @classmethod
-    def exists(cls, root):
-        """Return True if a table appears to exist at this OS path"""
-        return os.path.isfile(os.path.join(root, b"_format"))
-
-    @classmethod
-    def create(cls, root, layout, file_size, files_per_dir):
-        """Initialize a table at the given OS path with the
-        given layout string"""
-
-        # Calculate rows per file so that each file is approximately
-        # file_size bytes.
-        rkt = rocket.Rocket(layout, None)
-        rows_per_file = max(file_size // rkt.binary_size, 1)
-        rkt.close()
-
-        fmt = {
-            "rows_per_file": rows_per_file,
-            "files_per_dir": files_per_dir,
-            "layout": layout,
-            "version": 3
-        }
-        nilmdb.utils.atomic.replace_file(
-            os.path.join(root, b"_format"), pickle.dumps(fmt, 2))
-
-    # Normal methods
-    def __init__(self, root, initial_nrows=0):
-        """'root' is the full OS path to the directory of this table"""
-        self.root = root
-        self.initial_nrows = initial_nrows
-
-        # Load the format
-        with open(os.path.join(self.root, b"_format"), "rb") as f:
-            fmt = pickle.load(f)
-
-        if fmt["version"] != 3:
-            # Old versions used floating point timestamps, which aren't
-            # valid anymore.
-            raise NotImplementedError("old version " + str(fmt["version"]) +
-                                      " bulk data store is not supported")
-
-        self.rows_per_file = fmt["rows_per_file"]
-        self.files_per_dir = fmt["files_per_dir"]
-        self.layout = fmt["layout"]
-
-        # Use rocket to get row size and file size
-        rkt = rocket.Rocket(self.layout, None)
-        self.row_size = rkt.binary_size
-        self.file_size = rkt.binary_size * self.rows_per_file
-        rkt.close()
-
-        # Find nrows
-        self.nrows = self._get_nrows()
-
-    def close(self):
-        self.file_open.cache_remove_all()
-
-    # Internal helpers
-    def _get_nrows(self):
-        """Find nrows by locating the lexicographically last filename
-        and using its size"""
-        # Note that this just finds a 'nrows' that is guaranteed to be
-        # greater than the row number of any piece of data that
-        # currently exists, not necessarily all data that _ever_
-        # existed.
-        regex = re.compile(b"^[0-9a-f]{4,}$")
-
-        # Find the last directory.  We sort and loop through all of them,
-        # starting with the numerically greatest, because the dirs could be
-        # empty if something was deleted but the directory was unexpectedly
-        # not deleted.
-        subdirs = sorted(filter(regex.search, os.listdir(self.root)),
-                         key=lambda x: int(x, 16), reverse=True)
-
-        for subdir in subdirs:
-            # Now find the last file in that dir
-            path = os.path.join(self.root, subdir)
-            files = list(filter(regex.search, os.listdir(path)))
-            if not files:
-                # Empty dir: try the next one
-                continue
-
-            # Find the numerical max
-            filename = max(files, key=lambda x: int(x, 16))
-            offset = os.path.getsize(os.path.join(self.root, subdir, filename))
-
-            # Convert to row number
-            return self._row_from_offset(subdir, filename, offset)
-
-        # No files, so no data.  We typically start at row 0 in this
-        # case, although initial_nrows is specified during some tests
-        # to exercise other parts of the code better.  Since we have
-        # no files yet, round initial_nrows up so it points to a row
-        # that would begin a new file.
-        nrows = ((self.initial_nrows + (self.rows_per_file - 1)) //
-                 self.rows_per_file) * self.rows_per_file
-        return nrows
-
-    def _offset_from_row(self, row):
-        """Return a (subdir, filename, offset, count) tuple:
-
-          subdir: subdirectory for the file
-        filename: the filename that contains the specified row
-          offset: byte offset of the specified row within the file
-           count: number of rows (starting at offset) that fit in the file
-        """
-        filenum = row // self.rows_per_file
-        # It's OK if these format specifiers are too short; the filenames
-        # will just get longer but will still sort correctly.
-        dirname = sprintf(b"%04x", filenum // self.files_per_dir)
-        filename = sprintf(b"%04x", filenum % self.files_per_dir)
-        offset = (row % self.rows_per_file) * self.row_size
-        count = self.rows_per_file - (row % self.rows_per_file)
-        return (dirname, filename, offset, count)
-
-    def _row_from_offset(self, subdir, filename, offset):
-        """Return the row number that corresponds to the given
-        'subdir/filename' and byte-offset within that file."""
-        if (offset % self.row_size) != 0:
-            # this shouldn't occur, unless there is some corruption somewhere
-            raise ValueError("file offset is not a multiple of data size")
-        filenum = int(subdir, 16) * self.files_per_dir + int(filename, 16)
-        row = (filenum * self.rows_per_file) + (offset // self.row_size)
-        return row
-
-    def _remove_or_truncate_file(self, subdir, filename, offset=0):
-        """Remove the given file, and remove the subdirectory too
-        if it's empty.  If offset is nonzero, truncate the file
-        to that size instead."""
-        # Close potentially open file in file_open LRU cache
-        self.file_open.cache_remove(self, subdir, filename)
-        if offset:
-            # Truncate it
-            with open(os.path.join(self.root, subdir, filename), "r+b") as f:
-                f.truncate(offset)
-        else:
-            # Remove file
-            os.remove(os.path.join(self.root, subdir, filename))
-            # Try deleting subdir, too
-            try:
-                os.rmdir(os.path.join(self.root, subdir))
-            except Exception:
-                pass
-
-    # Cache open files
-    @nilmdb.utils.lru_cache(size=fd_cache_size,
-                            onremove=lambda f: f.close())
-    def file_open(self, subdir, filename):
-        """Open and map a given 'subdir/filename' (relative to self.root).
-        Will be automatically closed when evicted from the cache."""
-        # Create path if it doesn't exist
-        try:
-            os.mkdir(os.path.join(self.root, subdir))
-        except OSError:
-            pass
-        # Return a rocket.Rocket object, which contains the open file
-        return rocket.Rocket(self.layout,
-                             os.path.join(self.root, subdir, filename))
-
-    def append_data(self, data, start, end, binary=False):
-        """Parse the formatted string in 'data', according to the
-        current layout, and append it to the table.  If any timestamps
-        are non-monotonic, or don't fall between 'start' and 'end',
-        a ValueError is raised.
-
-        Note that data is always of 'bytes' type.
-
-        If 'binary' is True, the data should be in raw binary format
-        instead: little-endian, matching the current table's layout,
-        including the int64 timestamp.
-
-        If this function succeeds, it returns normally.  Otherwise,
-        the table is reverted back to its original state by truncating
-        or deleting files as necessary."""
-        data_offset = 0
-        last_timestamp = nilmdb.utils.time.min_timestamp
-        tot_rows = self.nrows
-        count = 0
-        linenum = 0
-        try:
-            while data_offset < len(data):
-                # See how many rows we can fit into the current file,
-                # and open it
-                (subdir, fname, offs, count) = self._offset_from_row(tot_rows)
-                f = self.file_open(subdir, fname)
-
-                # Ask the rocket object to parse and append up to "count"
-                # rows of data, verifying things along the way.
-                try:
-                    if binary:
-                        appender = f.append_binary
-                    else:
-                        appender = f.append_string
-                    (added_rows, data_offset, last_timestamp, linenum
-                     ) = appender(count, data, data_offset, linenum,
-                                  start, end, last_timestamp)
-                except rocket.ParseError as e:
-                    (linenum, colnum, errtype, obj) = e.args
-                    if binary:
-                        where = "byte %d: " % (linenum)
-                    else:
-                        where = "line %d, column %d: " % (linenum, colnum)
-                    # Extract out the error line, add column marker
-                    try:
-                        if binary:
-                            raise IndexError
-                        bad = data.splitlines()[linenum-1]
-                        bad += b'\n' + b' ' * (colnum - 1) + b'^'
-                    except IndexError:
-                        bad = b""
-                    if errtype == rocket.ERR_NON_MONOTONIC:
-                        err = "timestamp is not monotonically increasing"
-                    elif errtype == rocket.ERR_OUT_OF_INTERVAL:
-                        if obj < start:
-                            err = sprintf("Data timestamp %s < start time %s",
-                                          timestamp_to_string(obj),
-                                          timestamp_to_string(start))
-                        else:
-                            err = sprintf("Data timestamp %s >= end time %s",
-                                          timestamp_to_string(obj),
-                                          timestamp_to_string(end))
-                    else:
-                        err = str(obj)
-                    bad_str = bad.decode('utf-8', errors='backslashreplace')
-                    raise ValueError("error parsing input data: " +
-                                     where + err + "\n" + bad_str)
-                tot_rows += added_rows
-        except Exception:
-            # Some failure, so try to roll things back by truncating or
-            # deleting files that we may have appended data to.
-            cleanpos = self.nrows
-            while cleanpos <= tot_rows:
-                (subdir, fname, offs, count) = self._offset_from_row(cleanpos)
-                self._remove_or_truncate_file(subdir, fname, offs)
-                cleanpos += count
-            # Re-raise original exception
-            raise
-        else:
-            # Success, so update self.nrows accordingly
-            self.nrows = tot_rows
-
-    def get_data(self, start, stop, binary=False):
-        """Extract data corresponding to Python range [n:m],
-        and returns a formatted string"""
-        if (start is None or stop is None or
-                start > stop or start < 0 or stop > self.nrows):
-            raise IndexError("Index out of range")
-
-        ret = []
-        row = start
-        remaining = stop - start
-        while remaining > 0:
-            (subdir, filename, offset, count) = self._offset_from_row(row)
-            if count > remaining:
-                count = remaining
-            f = self.file_open(subdir, filename)
-            if binary:
-                ret.append(f.extract_binary(offset, count))
-            else:
-                ret.append(f.extract_string(offset, count))
-            remaining -= count
-            row += count
-        return b"".join(ret)
-
-    def __getitem__(self, row):
-        """Extract timestamps from a row, with table[n] notation."""
-        if row < 0 or row >= self.nrows:
-            raise IndexError("Index out of range")
-        (subdir, filename, offset, count) = self._offset_from_row(row)
-        f = self.file_open(subdir, filename)
-        return f.extract_timestamp(offset)
-
-    def _remove_rows(self, subdir, filename, start, stop):
-        """Helper to mark specific rows as being removed from a
-        file, and potentially remove or truncate the file itself."""
-        # Close potentially open file in file_open LRU cache
-        self.file_open.cache_remove(self, subdir, filename)
-
-        # We keep a file like 0000.removed that contains a list of
-        # which rows have been "removed".  Note that we never have to
-        # remove entries from this list, because we never decrease
-        # self.nrows, and so we will never overwrite those locations in the
-        # file.  Only when the list covers the entire extent of the
-        # file will that file be removed.
-        datafile = os.path.join(self.root, subdir, filename)
-        cachefile = datafile + b".removed"
-        try:
-            with open(cachefile, "rb") as f:
-                ranges = pickle.load(f)
-            cachefile_present = True
-        except Exception:
-            ranges = []
-            cachefile_present = False
-
-        # Append our new range and sort
-        ranges.append((start, stop))
-        ranges.sort()
-
-        # Merge adjacent ranges into "out"
-        merged = []
-        prev = None
-        for new in ranges:
-            if prev is None:
-                # No previous range, so remember this one
-                prev = new
-            elif prev[1] == new[0]:
-                # Previous range connected to this new one; extend prev
-                prev = (prev[0], new[1])
-            else:
-                # Not connected; append previous and start again
-                merged.append(prev)
-                prev = new
-        # Last range we were looking at goes into the file.  We know
-        # there was at least one (the one we just removed).
-        merged.append(prev)
-
-        # If the range covered the whole file, we can delete it now.
-        # Note that the last file in a table may be only partially
-        # full (smaller than self.rows_per_file).  We purposely leave
-        # those files around rather than deleting them, because the
-        # remainder will be filled on a subsequent append(), and things
-        # are generally easier if we don't have to special-case that.
-        if (len(merged) == 1 and
-                merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
-            # Delete files
-            if cachefile_present:
-                os.remove(cachefile)
-            self._remove_or_truncate_file(subdir, filename, 0)
-        else:
-            # File needs to stick around.  This means we can get
-            # degenerate cases where we have large files containing as
-            # little as one row.  Try to punch a hole in the file,
-            # so that this region doesn't take up filesystem space.
-            offset = start * self.row_size
-            count = (stop - start) * self.row_size
-            nilmdb.utils.fallocate.punch_hole(datafile, offset, count)
-
-            # Update cache.  Try to do it atomically.
-            nilmdb.utils.atomic.replace_file(cachefile,
-                                             pickle.dumps(merged, 2))
-
-    def remove(self, start, stop):
-        """Remove specified rows [start, stop) from this table.
-
-        If a file is left empty, it is fully removed.  Otherwise, a
-        parallel data file is used to remember which rows have been
-        removed, and the file is otherwise untouched."""
-        if start < 0 or start > stop or stop > self.nrows:
-            raise IndexError("Index out of range")
-
-        row = start
-        remaining = stop - start
-        while remaining:
-            # Loop through each file that we need to touch
-            (subdir, filename, offset, count) = self._offset_from_row(row)
-            if count > remaining:
-                count = remaining
-            row_offset = offset // self.row_size
-            # Mark the rows as being removed
-            self._remove_rows(subdir, filename, row_offset, row_offset + count)
-            remaining -= count
-            row += count

nilmdb/server/errors.py

@@ -1,15 +0,0 @@
-"""Exceptions"""
-
-
-class NilmDBError(Exception):
-    """Base exception for NilmDB errors"""
-    def __init__(self, msg="Unspecified error"):
-        super().__init__(msg)
-
-
-class StreamError(NilmDBError):
-    pass
-
-
-class OverlapError(NilmDBError):
-    pass

nilmdb/server/interval.pyx

@@ -1,329 +0,0 @@
-# cython: language_level=2
-
-"""Interval, IntervalSet
-
-The Interval implemented here is just like
-nilmdb.utils.interval.Interval, except implemented in Cython for
-speed.
-
-Represents an interval of time, and a set of such intervals.
-
-Intervals are half-open, ie. they include data points with timestamps
-[start, end)
-"""
-
-# First implementation kept a sorted list of intervals and used
-# biesct() to optimize some operations, but this was too slow.
-
-# Second version was based on the quicksect implementation from
-# python-bx, modified slightly to handle floating point intervals.
-# This didn't support deletion.
-
-# Third version is more similar to the first version, using a rb-tree
-# instead of a simple sorted list to maintain O(log n) operations.
-
-# Fourth version is an optimized rb-tree that stores interval starts
-# and ends directly in the tree, like bxinterval did.
-
-from ..utils.time import min_timestamp as nilmdb_min_timestamp
-from ..utils.time import max_timestamp as nilmdb_max_timestamp
-from ..utils.time import timestamp_to_string
-from ..utils.iterator import imerge
-from ..utils.interval import IntervalError
-import itertools
-
-cimport rbtree
-from libc.stdint cimport uint64_t, int64_t
-
-ctypedef int64_t timestamp_t
-
-cdef class Interval:
-    """Represents an interval of time."""
-
-    cdef public timestamp_t start, end
-
-    def __init__(self, timestamp_t start, timestamp_t end):
-        """
-        'start' and 'end' are arbitrary numbers that represent time
-        """
-        if start >= end:
-            # Explicitly disallow zero-width intervals (since they're half-open)
-            raise IntervalError("start %s must precede end %s" % (start, end))
-        self.start = start
-        self.end = end
-
-    def __repr__(self):
-        s = repr(self.start) + ", " + repr(self.end)
-        return self.__class__.__name__ + "(" + s + ")"
-
-    def __str__(self):
-        return ("[" + timestamp_to_string(self.start) +
-                " -> " + timestamp_to_string(self.end) + ")")
-
-    # Compare two intervals.  If non-equal, order by start then end
-    def __lt__(self, Interval other):
-        return (self.start, self.end) < (other.start, other.end)
-    def __gt__(self, Interval other):
-        return (self.start, self.end) > (other.start, other.end)
-    def __le__(self, Interval other):
-        return (self.start, self.end) <= (other.start, other.end)
-    def __ge__(self, Interval other):
-        return (self.start, self.end) >= (other.start, other.end)
-    def __eq__(self, Interval other):
-        return (self.start, self.end) == (other.start, other.end)
-    def __ne__(self, Interval other):
-        return (self.start, self.end) != (other.start, other.end)
-
-    cpdef intersects(self, Interval other):
-        """Return True if two Interval objects intersect"""
-        if (self.end <= other.start or self.start >= other.end):
-            return False
-        return True
-
-    cpdef subset(self, timestamp_t start, timestamp_t end):
-        """Return a new Interval that is a subset of this one"""
-        # A subclass that tracks additional data might override this.
-        if start < self.start or end > self.end:
-            raise IntervalError("not a subset")
-        return Interval(start, end)
-
-cdef class DBInterval(Interval):
-    """
-    Like Interval, but also tracks corresponding start/end times and
-    positions within the database.  These are not currently modified
-    when subsets are taken, but can be used later to help zero in on
-    database positions.
-
-    The actual 'start' and 'end' will always fall within the database
-    start and end, e.g.:
-        db_start = 100, db_startpos = 10000
-        start = 123
-        end = 150
-        db_end = 200, db_endpos = 20000
-    """
-
-    cpdef public timestamp_t db_start, db_end
-    cpdef public uint64_t db_startpos, db_endpos
-
-    def __init__(self, start, end,
-                 db_start, db_end,
-                 db_startpos, db_endpos):
-        """
-        'db_start' and 'db_end' are arbitrary numbers that represent
-        time.  They must be a strict superset of the time interval
-        covered by 'start' and 'end'.  The 'db_startpos' and
-        'db_endpos' are arbitrary database position indicators that
-        correspond to those points.
-        """
-        Interval.__init__(self, start, end)
-        self.db_start = db_start
-        self.db_end = db_end
-        self.db_startpos = db_startpos
-        self.db_endpos = db_endpos
-        if db_start > start or db_end < end:
-            raise IntervalError("database times must span the interval times")
-
-    def __repr__(self):
-        s = repr(self.start) + ", " + repr(self.end)
-        s += ", " + repr(self.db_start) + ", " + repr(self.db_end)
-        s += ", " + repr(self.db_startpos) + ", " + repr(self.db_endpos)
-        return self.__class__.__name__ + "(" + s + ")"
-
-    cpdef subset(self, timestamp_t start, timestamp_t end):
-        """
-        Return a new DBInterval that is a subset of this one
-        """
-        if start < self.start or end > self.end:
-            raise IntervalError("not a subset")
-        return DBInterval(start, end,
-                          self.db_start, self.db_end,
-                          self.db_startpos, self.db_endpos)
-
-cdef class IntervalSet:
-    """
-    A non-intersecting set of intervals.
-    """
-
-    cdef public rbtree.RBTree tree
-
-    def __init__(self, source=None):
-        """
-        'source' is an Interval or IntervalSet to add.
-        """
-        self.tree = rbtree.RBTree()
-        if source is not None:
-            self += source
-
-    def __iter__(self):
-        for node in self.tree:
-            if node.obj:
-                yield node.obj
-
-    def __len__(self):
-        return sum(1 for x in self)
-
-    def __repr__(self):
-        descs = [ repr(x) for x in self ]
-        return self.__class__.__name__ + "([" + ", ".join(descs) + "])"
-
-    def __str__(self):
-        descs = [ str(x) for x in self ]
-        return  "[" + ", ".join(descs) + "]"
-
-    def __match__(self, other):
-        # This isn't particularly efficient, but it shouldn't get used in the
-        # general case.
-        """Test equality of two IntervalSets.
-
-        Treats adjacent Intervals as equivalent to one long interval,
-        so this function really tests whether the IntervalSets cover
-        the same spans of time."""
-        i = 0
-        j = 0
-        outside = True
-
-        def is_adjacent(a, b):
-            """Return True if two Intervals are adjacent (same end or start)"""
-            if a.end == b.start or b.end == a.start:
-                return True
-            else:
-                return False
-
-        this = list(self)
-        that = list(other)
-
-        try:
-            while True:
-                if (outside):
-                    # To match, we need to be finished both sets
-                    if (i >= len(this) and j >= len(that)):
-                        return True
-                    # Or the starts need to match
-                    if (this[i].start != that[j].start):
-                        return False
-                    outside = False
-                else:
-                    # We can move on if the two interval ends match
-                    if (this[i].end == that[j].end):
-                        i += 1
-                        j += 1
-                        outside = True
-                    else:
-                        # Whichever ends first needs to be adjacent to the next
-                        if (this[i].end < that[j].end):
-                            if (not is_adjacent(this[i],this[i+1])):
-                                return False
-                            i += 1
-                        else:
-                            if (not is_adjacent(that[j],that[j+1])):
-                                return False
-                            j += 1
-        except IndexError:
-            return False
-
-    # Use __richcmp__ instead of __eq__, __ne__ for Cython.
-    def __richcmp__(self, other, int op):
-        if op == 2: # ==
-            return self.__match__(other)
-        elif op == 3: # !=
-            return not self.__match__(other)
-        return False
-    #def __eq__(self, other):
-    #    return self.__match__(other)
-    #
-    #def __ne__(self, other):
-    #    return not self.__match__(other)
-
-    def __iadd__(self, object other not None):
-        """Inplace add -- modifies self
-
-        This throws an exception if the regions being added intersect."""
-        if isinstance(other, Interval):
-            if self.intersects(other):
-                raise IntervalError("Tried to add overlapping interval "
-                                    "to this set")
-            self.tree.insert(rbtree.RBNode(other.start, other.end, other))
-        else:
-            for x in other:
-                self.__iadd__(x)
-        return self
-
-    def iadd_nocheck(self, Interval other not None):
-        """Inplace add -- modifies self.
-        'Optimized' version that doesn't check for intersection and
-        only inserts the new interval into the tree."""
-        self.tree.insert(rbtree.RBNode(other.start, other.end, other))
-
-    def __isub__(self, Interval other not None):
-        """Inplace subtract -- modifies self
-
-        Removes an interval from the set.  Must exist exactly
-        as provided -- cannot remove a subset of an existing interval."""
-        i = self.tree.find(other.start, other.end)
-        if i is None:
-            raise IntervalError("interval " + str(other) + " not in tree")
-        self.tree.delete(i)
-        return self
-
-    def __add__(self, other not None):
-        """Add -- returns a new object"""
-        new = IntervalSet(self)
-        new += IntervalSet(other)
-        return new
-
-    def __and__(self, other not None):
-        """
-        Compute a new IntervalSet from the intersection of this
-        IntervalSet with one other interval.
-
-        Output intervals are built as subsets of the intervals in the
-        first argument (self).
-        """
-        out = IntervalSet()
-        for i in self.intersection(other):
-            out.tree.insert(rbtree.RBNode(i.start, i.end, i))
-        return out
-
-    def intersection(self, Interval interval not None, orig = False):
-        """
-        Compute a sequence of intervals that correspond to the
-        intersection between `self` and the provided interval.
-        Returns a generator that yields each of these intervals
-        in turn.
-
-        Output intervals are built as subsets of the intervals in the
-        first argument (self).
-
-        If orig = True, also return the original interval that was
-        (potentially) subsetted to make the one that is being
-        returned.
-        """
-        if orig:
-            for n in self.tree.intersect(interval.start, interval.end):
-                i = n.obj
-                subset = i.subset(max(i.start, interval.start),
-                                  min(i.end, interval.end))
-                yield (subset, i)
-        else:
-            for n in self.tree.intersect(interval.start, interval.end):
-                i = n.obj
-                subset = i.subset(max(i.start, interval.start),
-                                  min(i.end, interval.end))
-                yield subset
-
-    cpdef intersects(self, Interval other):
-        """Return True if this IntervalSet intersects another interval"""
-        for n in self.tree.intersect(other.start, other.end):
-            if n.obj.intersects(other):
-                return True
-        return False
-
-    def find_end(self, timestamp_t t):
-        """
-        Return an Interval from this tree that ends at time t, or
-        None if it doesn't exist.
-        """
-        n = self.tree.find_left_end(t)
-        if n and n.obj.end == t:
-            return n.obj
-        return None

nilmdb/server/interval.pyxdep

@@ -1 +0,0 @@
-rbtree.pxd

nilmdb/server/nilmdb.py

@@ -1,717 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""NilmDB
-
-Object that represents a NILM database file.
-
-Manages both the SQL database and the table storage backend.
-"""
-
-import os
-import errno
-import sqlite3
-
-import nilmdb.utils
-from nilmdb.utils.printf import printf
-from nilmdb.utils.time import timestamp_to_bytes
-
-from nilmdb.utils.interval import IntervalError
-from nilmdb.server.interval import Interval, DBInterval, IntervalSet
-
-from nilmdb.server import bulkdata
-from nilmdb.server.errors import NilmDBError, StreamError, OverlapError
-
-# Note about performance and transactions:
-#
-# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
-# takes about 125msec.  sqlite3 will commit transactions at 3 times:
-# 1: explicit con.commit()
-# 2: between a series of DML commands and non-DML commands, e.g.
-#    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
-# 3: at the end of an explicit transaction, e.g. "with self.con as con:"
-#
-# To speed things up, we can set 'PRAGMA synchronous=OFF'.  Or, it
-# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL'
-# give an equivalent speedup more safely.  That is what is used here.
-_sql_schema_updates = {
-    0: {"next": 1, "sql": """
-    -- All streams
-    CREATE TABLE streams(
-        id INTEGER PRIMARY KEY,		-- stream ID
-        path TEXT UNIQUE NOT NULL,	-- path, e.g. '/newton/prep'
-        layout TEXT NOT NULL		-- layout name, e.g. float32_8
-    );
-
-    -- Individual timestamped ranges in those streams.
-    -- For a given start_time and end_time, this tells us that the
-    -- data is stored between start_pos and end_pos.
-    -- Times are stored as μs since Unix epoch
-    -- Positions are opaque: PyTables rows, file offsets, etc.
-    --
-    -- Note: end_pos points to the row _after_ end_time, so end_pos-1
-    -- is the last valid row.
-    CREATE TABLE ranges(
-        stream_id INTEGER NOT NULL,
-        start_time INTEGER NOT NULL,
-        end_time INTEGER NOT NULL,
-        start_pos INTEGER NOT NULL,
-        end_pos INTEGER NOT NULL
-    );
-    CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time);
-    """},
-
-    1: {"next": 3, "sql": """
-    -- Generic dictionary-type metadata that can be associated with a stream
-    CREATE TABLE metadata(
-        stream_id INTEGER NOT NULL,
-        key TEXT NOT NULL,
-        value TEXT
-    );
-    """},
-
-    2: {"error": "old format with floating-point timestamps requires "
-        "nilmdb 1.3.1 or older"},
-
-    3: {"next": None},
-}
-
-
-@nilmdb.utils.must_close()
-class NilmDB():
-    verbose = 0
-
-    def __init__(self, basepath,
-                 max_results=None,
-                 max_removals=None,
-                 max_int_removals=None,
-                 bulkdata_args=None):
-        """Initialize NilmDB at the given basepath.
-        Other arguments are for debugging / testing:
-
-        'max_results' is the max rows to send in a single
-        stream_intervals or stream_extract response.
-
-        'max_removals' is the max rows to delete at once
-        in stream_remove.
-
-        'max_int_removals' is the max intervals to delete
-        at once in stream_remove.
-
-        'bulkdata_args' is kwargs for the bulkdata module.
-        """
-        if bulkdata_args is None:
-            bulkdata_args = {}
-
-        # set up path
-        self.basepath = os.path.abspath(basepath)
-
-        # Create the database path if it doesn't exist
-        try:
-            os.makedirs(self.basepath)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise IOError("can't create tree " + self.basepath)
-
-        # Our data goes inside it
-        self.data = bulkdata.BulkData(self.basepath, **bulkdata_args)
-
-        # SQLite database too
-        sqlfilename = os.path.join(self.basepath, "data.sql")
-        self.con = sqlite3.connect(sqlfilename, check_same_thread=True)
-        try:
-            self._sql_schema_update()
-        except Exception:
-            self.data.close()
-            raise
-
-        # See big comment at top about the performance implications of this
-        self.con.execute("PRAGMA synchronous=NORMAL")
-        self.con.execute("PRAGMA journal_mode=WAL")
-
-        # Approximate largest number of elements that we want to send
-        # in a single reply (for stream_intervals, stream_extract).
-        self.max_results = max_results or 16384
-
-        # Remove up to this many rows per call to stream_remove.
-        self.max_removals = max_removals or 1048576
-
-        # Remove up to this many intervals per call to stream_remove.
-        self.max_int_removals = max_int_removals or 4096
-
-    def get_basepath(self):
-        return self.basepath
-
-    def close(self):
-        if self.con:
-            self.con.commit()
-            self.con.close()
-            self.con = None
-        self.data.close()
-
-    def _sql_schema_update(self):
-        cur = self.con.cursor()
-        version = cur.execute("PRAGMA user_version").fetchone()[0]
-        oldversion = version
-
-        while True:
-            if version not in _sql_schema_updates:
-                raise Exception(self.basepath + ": unknown database version "
-                                + str(version))
-            update = _sql_schema_updates[version]
-            if "error" in update:
-                raise Exception(self.basepath + ": can't use database version "
-                                + str(version) + ": " + update["error"])
-            if update["next"] is None:
-                break
-            cur.executescript(update["sql"])
-            version = update["next"]
-            if self.verbose:
-                printf("Database schema updated to %d\n", version)
-
-        if version != oldversion:
-            with self.con:
-                cur.execute("PRAGMA user_version = {v:d}".format(v=version))
-
-    def _check_user_times(self, start, end):
-        if start is None:
-            start = nilmdb.utils.time.min_timestamp
-        if end is None:
-            end = nilmdb.utils.time.max_timestamp
-        if start >= end:
-            raise NilmDBError("start must precede end")
-        return (start, end)
-
-    @nilmdb.utils.lru_cache(size=64)
-    def _get_intervals(self, stream_id):
-        """
-        Return a mutable IntervalSet corresponding to the given stream ID.
-        """
-        iset = IntervalSet()
-        result = self.con.execute("SELECT start_time, end_time, "
-                                  "start_pos, end_pos "
-                                  "FROM ranges "
-                                  "WHERE stream_id=?", (stream_id,))
-        try:
-            for (start_time, end_time, start_pos, end_pos) in result:
-                iset += DBInterval(start_time, end_time,
-                                   start_time, end_time,
-                                   start_pos, end_pos)
-        except IntervalError:
-            raise NilmDBError("unexpected overlap in ranges table!")
-
-        return iset
-
-    def _sql_interval_insert(self, id, start, end, start_pos, end_pos):
-        """Helper that adds interval to the SQL database only"""
-        self.con.execute("INSERT INTO ranges "
-                         "(stream_id,start_time,end_time,start_pos,end_pos) "
-                         "VALUES (?,?,?,?,?)",
-                         (id, start, end, start_pos, end_pos))
-
-    def _sql_interval_delete(self, id, start, end, start_pos, end_pos):
-        """Helper that removes interval from the SQL database only"""
-        self.con.execute("DELETE FROM ranges WHERE "
-                         "stream_id=? AND start_time=? AND "
-                         "end_time=? AND start_pos=? AND end_pos=?",
-                         (id, start, end, start_pos, end_pos))
-
-    def _add_interval(self, stream_id, interval, start_pos, end_pos):
-        """
-        Add interval to the internal interval cache, and to the database.
-        Note: arguments must be ints (not numpy.int64, etc)
-        """
-        # Load this stream's intervals
-        iset = self._get_intervals(stream_id)
-
-        # Check for adjacency.  If there's a stream in the database
-        # that ends exactly when this one starts, and the database
-        # rows match up, we can make one interval that covers the
-        # time range [adjacent.start -> interval.end)
-        # and database rows [ adjacent.start_pos -> end_pos ].
-        # Only do this if the resulting interval isn't too large.
-        max_merged_rows = 8000 * 60 * 60 * 1.05  # 1.05 hours at 8 KHz
-        adjacent = iset.find_end(interval.start)
-        if (adjacent is not None and
-                start_pos == adjacent.db_endpos and
-                (end_pos - adjacent.db_startpos) < max_merged_rows):
-            # First delete the old one, both from our iset and the
-            # database
-            iset -= adjacent
-            self._sql_interval_delete(stream_id,
-                                      adjacent.db_start, adjacent.db_end,
-                                      adjacent.db_startpos, adjacent.db_endpos)
-
-            # Now update our interval so the fallthrough add is
-            # correct.
-            interval.start = adjacent.start
-            start_pos = adjacent.db_startpos
-
-        # Add the new interval to the iset
-        iset.iadd_nocheck(DBInterval(interval.start, interval.end,
-                                     interval.start, interval.end,
-                                     start_pos, end_pos))
-
-        # Insert into the database
-        self._sql_interval_insert(stream_id, interval.start, interval.end,
-                                  int(start_pos), int(end_pos))
-
-        self.con.commit()
-
-    def _remove_interval(self, stream_id, original, remove):
-        """
-        Remove an interval from the internal cache and the database.
-
-        stream_id: id of stream
-         original: original DBInterval; must be already present in DB
-        to_remove: DBInterval to remove; must be subset of 'original'
-        """
-        # Load this stream's intervals
-        iset = self._get_intervals(stream_id)
-
-        # Remove existing interval from the cached set and the database
-        iset -= original
-        self._sql_interval_delete(stream_id,
-                                  original.db_start, original.db_end,
-                                  original.db_startpos, original.db_endpos)
-
-        # Add back the intervals that would be left over if the
-        # requested interval is removed.  There may be two of them, if
-        # the removed piece was in the middle.
-        def add(iset, start, end, start_pos, end_pos):
-            iset += DBInterval(start, end, start, end, start_pos, end_pos)
-            self._sql_interval_insert(stream_id, start, end,
-                                      start_pos, end_pos)
-
-        if original.start != remove.start:
-            # Interval before the removed region
-            add(iset, original.start, remove.start,
-                original.db_startpos, remove.db_startpos)
-
-        if original.end != remove.end:
-            # Interval after the removed region
-            add(iset, remove.end, original.end,
-                remove.db_endpos, original.db_endpos)
-
-        # Commit SQL changes
-        self.con.commit()
-
-        return
-
-    def stream_list(self, path=None, layout=None, extended=False):
-        """Return list of lists of all streams in the database.
-
-        If path is specified, include only streams with a path that
-        matches the given string.
-
-        If layout is specified, include only streams with a layout
-        that matches the given string.
-
-        If extended=False, returns a list of lists containing
-        the path and layout: [ path, layout ]
-
-        If extended=True, returns a list of lists containing
-        more information:
-           path
-           layout
-           interval_min (earliest interval start)
-           interval_max (latest interval end)
-           rows         (total number of rows of data)
-           time         (total time covered by this stream, in timestamp units)
-        """
-        params = ()
-        query = "SELECT streams.path, streams.layout"
-        if extended:
-            query += ", min(ranges.start_time), max(ranges.end_time) "
-            query += ", coalesce(sum(ranges.end_pos - ranges.start_pos), 0) "
-            query += ", coalesce(sum(ranges.end_time - ranges.start_time), 0) "
-        query += " FROM streams"
-        if extended:
-            query += " LEFT JOIN ranges ON streams.id = ranges.stream_id"
-        query += " WHERE 1=1"
-        if layout is not None:
-            query += " AND streams.layout=?"
-            params += (layout,)
-        if path is not None:
-            query += " AND streams.path=?"
-            params += (path,)
-        query += " GROUP BY streams.id ORDER BY streams.path"
-        result = self.con.execute(query, params).fetchall()
-        return [list(x) for x in result]
-
-    def stream_intervals(self, path, start=None, end=None, diffpath=None):
-        """
-        List all intervals in 'path' between 'start' and 'end'.  If
-        'diffpath' is not none, list instead the set-difference
-        between the intervals in the two streams; i.e. all interval
-        ranges that are present in 'path' but not 'diffpath'.
-
-        Returns (intervals, restart) tuple.
-
-        'intervals' is a list of [start,end] timestamps of all intervals
-        that exist for path, between start and end.
-
-        'restart', if not None, means that there were too many results
-        to return in a single request.  The data is complete from the
-        starting timestamp to the point at which it was truncated, and
-        a new request with a start time of 'restart' will fetch the
-        next block of data.
-        """
-        stream_id = self._stream_id(path)
-        intervals = self._get_intervals(stream_id)
-        if diffpath:
-            diffstream_id = self._stream_id(diffpath)
-            diffintervals = self._get_intervals(diffstream_id)
-        (start, end) = self._check_user_times(start, end)
-        requested = Interval(start, end)
-        result = []
-        if diffpath:
-            getter = nilmdb.utils.interval.set_difference(
-                intervals.intersection(requested),
-                diffintervals.intersection(requested))
-        else:
-            getter = intervals.intersection(requested)
-        for n, i in enumerate(getter):
-            if n >= self.max_results:
-                restart = i.start
-                break
-            result.append([i.start, i.end])
-        else:
-            restart = None
-        return (result, restart)
-
-    def stream_create(self, path, layout_name):
-        """Create a new table in the database.
-
-        path: path to the data (e.g. '/newton/prep').
-        Paths must contain at least two elements, e.g.:
-           /newton/prep
-           /newton/raw
-           /newton/upstairs/prep
-           /newton/upstairs/raw
-
-        layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
-        """
-        # Create the bulk storage.  Raises ValueError on error, which we
-        # pass along.
-        self.data.create(path, layout_name)
-
-        # Insert into SQL database once the bulk storage is happy
-        with self.con as con:
-            con.execute("INSERT INTO streams (path, layout) VALUES (?,?)",
-                        (path, layout_name))
-
-    def _stream_id(self, path):
-        """Return unique stream ID"""
-        result = self.con.execute("SELECT id FROM streams WHERE path=?",
-                                  (path,)).fetchone()
-        if result is None:
-            raise StreamError("No stream at path " + path)
-        return result[0]
-
-    def stream_set_metadata(self, path, data):
-        """Set stream metadata from a dictionary, e.g.
-           { description: 'Downstairs lighting',
-             v_scaling: 123.45 }
-           This replaces all existing metadata.
-           """
-        stream_id = self._stream_id(path)
-        with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
-            for key in data:
-                if data[key] != '':
-                    con.execute("INSERT INTO metadata VALUES (?, ?, ?)",
-                                (stream_id, key, data[key]))
-
-    def stream_get_metadata(self, path):
-        """Return stream metadata as a dictionary."""
-        stream_id = self._stream_id(path)
-        result = self.con.execute("SELECT metadata.key, metadata.value "
-                                  "FROM metadata "
-                                  "WHERE metadata.stream_id=?", (stream_id,))
-        data = {}
-        for (key, value) in result:
-            data[key] = value
-        return data
-
-    def stream_update_metadata(self, path, newdata):
-        """Update stream metadata from a dictionary"""
-        data = self.stream_get_metadata(path)
-        data.update(newdata)
-        self.stream_set_metadata(path, data)
-
-    def stream_rename(self, oldpath, newpath):
-        """Rename a stream."""
-        stream_id = self._stream_id(oldpath)
-
-        # Rename the data
-        self.data.rename(oldpath, newpath)
-
-        # Rename the stream in the database
-        with self.con as con:
-            con.execute("UPDATE streams SET path=? WHERE id=?",
-                        (newpath, stream_id))
-
-    def stream_destroy(self, path):
-        """Fully remove a table from the database.  Fails if there are
-        any intervals data present; remove them first.  Metadata is
-        also removed."""
-        stream_id = self._stream_id(path)
-
-        # Verify that no intervals are present, and clear the cache
-        iset = self._get_intervals(stream_id)
-        if iset:
-            raise NilmDBError("all intervals must be removed before "
-                              "destroying a stream")
-        self._get_intervals.cache_remove(self, stream_id)
-
-        # Delete the bulkdata storage
-        self.data.destroy(path)
-
-        # Delete metadata, stream, intervals (should be none)
-        with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
-
-    def stream_insert(self, path, start, end, data, binary=False):
-        """Insert new data into the database.
-           path: Path at which to add the data
-           start: Starting timestamp
-           end: Ending timestamp
-           data: Textual data, formatted according to the layout of path
-
-           'binary', if True, means that 'data' is raw binary:
-           little-endian, matching the current table's layout,
-           including the int64 timestamp.
-           """
-        # First check for basic overlap using timestamp info given.
-        stream_id = self._stream_id(path)
-        iset = self._get_intervals(stream_id)
-        interval = Interval(start, end)
-        if iset.intersects(interval):
-            raise OverlapError("new data overlaps existing data at range: "
-                               + str(iset & interval))
-
-        # Tenatively append the data.  This will raise a ValueError if
-        # there are any parse errors.
-        table = self.data.getnode(path)
-        row_start = table.nrows
-        table.append_data(data, start, end, binary)
-        row_end = table.nrows
-
-        # Insert the record into the sql database.
-        self._add_interval(stream_id, interval, row_start, row_end)
-
-        # And that's all
-        return
-
-    def _bisect_left(self, a, x, lo, hi):
-        # Like bisect.bisect_left, but doesn't choke on large indices on
-        # 32-bit systems, like bisect's fast C implementation does.
-        while lo < hi:
-            mid = (lo + hi) // 2
-            if a[mid] < x:
-                lo = mid + 1
-            else:
-                hi = mid
-        return lo
-
-    def _find_start(self, table, dbinterval):
-        """
-        Given a DBInterval, find the row in the database that
-        corresponds to the start time.  Return the first database
-        position with a timestamp (first element) greater than or
-        equal to 'start'.
-        """
-        # Optimization for the common case where an interval wasn't truncated
-        if dbinterval.start == dbinterval.db_start:
-            return dbinterval.db_startpos
-        return self._bisect_left(table,
-                                 dbinterval.start,
-                                 dbinterval.db_startpos,
-                                 dbinterval.db_endpos)
-
-    def _find_end(self, table, dbinterval):
-        """
-        Given a DBInterval, find the row in the database that follows
-        the end time.  Return the first database position after the
-        row with timestamp (first element) greater than or equal
-        to 'end'.
-        """
-        # Optimization for the common case where an interval wasn't truncated
-        if dbinterval.end == dbinterval.db_end:
-            return dbinterval.db_endpos
-        # Note that we still use bisect_left here, because we don't
-        # want to include the given timestamp in the results.  This is
-        # so a queries like 1:00 -> 2:00 and 2:00 -> 3:00 return
-        # non-overlapping data.
-        return self._bisect_left(table,
-                                 dbinterval.end,
-                                 dbinterval.db_startpos,
-                                 dbinterval.db_endpos)
-
-    def stream_extract(self, path, start=None, end=None,
-                       count=False, markup=False, binary=False):
-        """
-        Returns (data, restart) tuple.
-
-        'data' is ASCII-formatted data from the database, formatted
-        according to the layout of the stream.
-
-        'restart', if not None, means that there were too many results to
-        return in a single request.  The data is complete from the
-        starting timestamp to the point at which it was truncated,
-        and a new request with a start time of 'restart' will fetch
-        the next block of data.
-
-        'count', if true, means to not return raw data, but just the count
-        of rows that would have been returned.  This is much faster
-        than actually fetching the data.  It is not limited by
-        max_results.
-
-        'markup', if true, indicates that returned data should be
-        marked with a comment denoting when a particular interval
-        starts, and another comment when an interval ends.
-
-        'binary', if true, means to return raw binary rather than
-        ASCII-formatted data.
-        """
-        stream_id = self._stream_id(path)
-        table = self.data.getnode(path)
-        intervals = self._get_intervals(stream_id)
-        (start, end) = self._check_user_times(start, end)
-        requested = Interval(start, end)
-        result = []
-        matched = 0
-        remaining = self.max_results
-        restart = None
-        if binary and (markup or count):
-            raise NilmDBError("binary mode can't be used with markup or count")
-        for interval in intervals.intersection(requested):
-            # Reading single rows from the table is too slow, so
-            # we use two bisections to find both the starting and
-            # ending row for this particular interval, then
-            # read the entire range as one slice.
-            row_start = self._find_start(table, interval)
-            row_end = self._find_end(table, interval)
-
-            if count:
-                matched += row_end - row_start
-                continue
-
-            # Shorten it if we'll hit the maximum number of results
-            row_max = row_start + remaining
-            if row_max < row_end:
-                row_end = row_max
-                restart = table[row_max]
-
-            # Add markup
-            if markup:
-                result.append(b"# interval-start " +
-                              timestamp_to_bytes(interval.start) + b"\n")
-
-            # Gather these results up
-            result.append(table.get_data(row_start, row_end, binary))
-
-            # Count them
-            remaining -= row_end - row_start
-
-            # Add markup, and exit if restart is set.
-            if restart is not None:
-                if markup:
-                    result.append(b"# interval-end " +
-                                  timestamp_to_bytes(restart) + b"\n")
-                break
-            if markup:
-                result.append(b"# interval-end " +
-                              timestamp_to_bytes(interval.end) + b"\n")
-
-        if count:
-            return matched
-        full_result = b"".join(result)
-        return (full_result, restart)
-
-    def stream_remove(self, path, start=None, end=None):
-        """
-        Remove data from the specified time interval within a stream.
-
-        Removes data in the interval [start, end), and intervals are
-        truncated or split appropriately.
-
-        Returns a (removed, restart) tuple.
-
-        'removed' is the number of data points that were removed.
-
-        'restart', if not None, means there were too many rows to
-        remove in a single request.  This function should be called
-        again with a start time of 'restart' to complete the removal.
-        """
-        stream_id = self._stream_id(path)
-        table = self.data.getnode(path)
-        intervals = self._get_intervals(stream_id)
-        (start, end) = self._check_user_times(start, end)
-        to_remove = Interval(start, end)
-        removed = 0
-        remaining = self.max_removals
-        int_remaining = self.max_int_removals
-        restart = None
-
-        # Can't remove intervals from within the iterator, so we need to
-        # remember what's currently in the intersection now.
-        all_candidates = list(intervals.intersection(to_remove, orig=True))
-
-        remove_start = None
-        remove_end = None
-
-        for (dbint, orig) in all_candidates:
-            # Stop if we've hit the max number of interval removals
-            if int_remaining <= 0:
-                restart = dbint.start
-                break
-
-            # Find row start and end
-            row_start = self._find_start(table, dbint)
-            row_end = self._find_end(table, dbint)
-
-            # Shorten it if we'll hit the maximum number of removals
-            row_max = row_start + remaining
-            if row_max < row_end:
-                row_end = row_max
-                dbint.end = table[row_max]
-                restart = dbint.end
-
-            # Adjust the DBInterval to match the newly found ends
-            dbint.db_start = dbint.start
-            dbint.db_end = dbint.end
-            dbint.db_startpos = row_start
-            dbint.db_endpos = row_end
-
-            # Remove interval from the database
-            self._remove_interval(stream_id, orig, dbint)
-
-            # Remove data from the underlying table storage,
-            # coalescing adjacent removals to reduce the number of calls
-            # to table.remove.
-            if remove_end == row_start:
-                # Extend our coalesced region
-                remove_end = row_end
-            else:
-                # Perform previous removal, then save this one
-                if remove_end is not None:
-                    table.remove(remove_start, remove_end)
-                remove_start = row_start
-                remove_end = row_end
-
-            # Count how many were removed
-            removed += row_end - row_start
-            remaining -= row_end - row_start
-            int_remaining -= 1
-
-            if restart is not None:
-                break
-
-        # Perform any final coalesced removal
-        if remove_end is not None:
-            table.remove(remove_start, remove_end)
-
-        return (removed, restart)

nilmdb/server/rbtree.pxd

@@ -1,25 +0,0 @@
-# cython: language_level=2
-
-cdef class RBNode:
-    cdef public object obj
-    cdef public double start, end
-    cdef public int red
-    cdef public RBNode left, right, parent
-
-cdef class RBTree:
-    cdef public RBNode nil, root
-
-    cpdef getroot(RBTree self)
-    cdef void __rotate_left(RBTree self, RBNode x)
-    cdef void __rotate_right(RBTree self, RBNode y)
-    cdef RBNode __successor(RBTree self, RBNode x)
-    cpdef RBNode successor(RBTree self, RBNode x)
-    cdef RBNode __predecessor(RBTree self, RBNode x)
-    cpdef RBNode predecessor(RBTree self, RBNode x)
-    cpdef insert(RBTree self, RBNode z)
-    cdef void __insert_fixup(RBTree self, RBNode x)
-    cpdef delete(RBTree self, RBNode z)
-    cdef inline void __delete_fixup(RBTree self, RBNode x)
-    cpdef RBNode find(RBTree self, double start, double end)
-    cpdef RBNode find_left_end(RBTree self, double t)
-    cpdef RBNode find_right_start(RBTree self, double t)

nilmdb/server/rbtree.pyx

@@ -1,378 +0,0 @@
-# cython: profile=False
-# cython: cdivision=True
-# cython: language_level=2
-
-"""
-Jim Paris <jim@jtan.com>
-
-Red-black tree, where keys are stored as start/end timestamps.
-This is a basic interval tree that holds half-open intervals:
-  [start, end)
-Intervals must not overlap.  Fixing that would involve making this
-into an augmented interval tree as described in CLRS 14.3.
-
-Code that assumes non-overlapping intervals is marked with the
-string 'non-overlapping'.
-"""
-
-import sys
-cimport rbtree
-
-cdef class RBNode:
-    """One node of the Red/Black tree, containing a key (start, end)
-    and value (obj)"""
-    def __init__(self, double start, double end, object obj = None):
-        self.obj = obj
-        self.start = start
-        self.end = end
-        self.red = False
-        self.left = None
-        self.right = None
-
-    def __str__(self):
-        if self.red:
-            color = "R"
-        else:
-            color = "B"
-        if self.start == sys.float_info.min:
-            return "[node nil]"
-        return ("[node ("
-                + str(self.obj) + ") "
-                + str(self.start) + " -> " + str(self.end) + " "
-                + color + "]")
-
-cdef class RBTree:
-    """Red/Black tree"""
-
-    # Init
-    def __init__(self):
-        self.nil = RBNode(start = sys.float_info.min,
-                          end = sys.float_info.min)
-        self.nil.left = self.nil
-        self.nil.right = self.nil
-        self.nil.parent = self.nil
-
-        self.root = RBNode(start = sys.float_info.max,
-                           end = sys.float_info.max)
-        self.root.left = self.nil
-        self.root.right = self.nil
-        self.root.parent = self.nil
-
-    # We have a dummy root node to simplify operations, so from an
-    # external point of view, its left child is the real root.
-    cpdef getroot(self):
-        return self.root.left
-
-    # Rotations and basic operations
-    cdef void __rotate_left(self, RBNode x):
-        """Rotate left:
-        #   x           y
-        #  / \   -->   / \
-        # z   y       x   w
-        #    / \     / \
-        #   v   w   z   v
-        """
-        cdef RBNode y = x.right
-        x.right = y.left
-        if y.left is not self.nil:
-            y.left.parent = x
-        y.parent = x.parent
-        if x is x.parent.left:
-            x.parent.left = y
-        else:
-            x.parent.right = y
-        y.left = x
-        x.parent = y
-
-    cdef void __rotate_right(self, RBNode y):
-        """Rotate right:
-        #     y           x
-        #    / \   -->   / \
-        #   x   w       z   y
-        #  / \             / \
-        # z   v           v   w
-        """
-        cdef RBNode x = y.left
-        y.left = x.right
-        if x.right is not self.nil:
-            x.right.parent = y
-        x.parent = y.parent
-        if y is y.parent.left:
-            y.parent.left = x
-        else:
-            y.parent.right = x
-        x.right = y
-        y.parent = x
-
-    cdef RBNode __successor(self, RBNode x):
-        """Returns the successor of RBNode x"""
-        cdef RBNode y = x.right
-        if y is not self.nil:
-            while y.left is not self.nil:
-                y = y.left
-        else:
-            y = x.parent
-            while x is y.right:
-                x = y
-                y = y.parent
-            if y is self.root:
-                return self.nil
-        return y
-    cpdef RBNode successor(self, RBNode x):
-        """Returns the successor of RBNode x, or None"""
-        cdef RBNode y = self.__successor(x)
-        return y if y is not self.nil else None
-
-    cdef RBNode __predecessor(self, RBNode x):
-        """Returns the predecessor of RBNode x"""
-        cdef RBNode y = x.left
-        if y is not self.nil:
-            while y.right is not self.nil:
-                y = y.right
-        else:
-            y = x.parent
-            while x is y.left:
-                if y is self.root:
-                    y = self.nil
-                    break
-                x = y
-                y = y.parent
-        return y
-    cpdef RBNode predecessor(self, RBNode x):
-        """Returns the predecessor of RBNode x, or None"""
-        cdef RBNode y = self.__predecessor(x)
-        return y if y is not self.nil else None
-
-    # Insertion
-    cpdef insert(self, RBNode z):
-        """Insert RBNode z into RBTree and rebalance as necessary"""
-        z.left = self.nil
-        z.right = self.nil
-        cdef RBNode y = self.root
-        cdef RBNode x = self.root.left
-        while x is not self.nil:
-            y = x
-            if (x.start > z.start or (x.start == z.start and x.end > z.end)):
-                x = x.left
-            else:
-                x = x.right
-        z.parent = y
-        if (y is self.root or
-            (y.start > z.start or (y.start == z.start and y.end > z.end))):
-            y.left = z
-        else:
-            y.right = z
-        # relabel/rebalance
-        self.__insert_fixup(z)
-
-    cdef void __insert_fixup(self, RBNode x):
-        """Rebalance/fix RBTree after a simple insertion of RBNode x"""
-        x.red = True
-        while x.parent.red:
-            if x.parent is x.parent.parent.left:
-                y = x.parent.parent.right
-                if y.red:
-                    x.parent.red = False
-                    y.red = False
-                    x.parent.parent.red = True
-                    x = x.parent.parent
-                else:
-                    if x is x.parent.right:
-                        x = x.parent
-                        self.__rotate_left(x)
-                    x.parent.red = False
-                    x.parent.parent.red = True
-                    self.__rotate_right(x.parent.parent)
-            else: # same as above, left/right switched
-                y = x.parent.parent.left
-                if y.red:
-                    x.parent.red = False
-                    y.red = False
-                    x.parent.parent.red = True
-                    x = x.parent.parent
-                else:
-                    if x is x.parent.left:
-                        x = x.parent
-                        self.__rotate_right(x)
-                    x.parent.red = False
-                    x.parent.parent.red = True
-                    self.__rotate_left(x.parent.parent)
-        self.root.left.red = False
-
-    # Deletion
-    cpdef delete(self, RBNode z):
-        if z.left is None or z.right is None:
-            raise AttributeError("you can only delete a node object "
-                                 + "from the tree; use find() to get one")
-        cdef RBNode x, y
-        if z.left is self.nil or z.right is self.nil:
-            y = z
-        else:
-            y = self.__successor(z)
-        if y.left is self.nil:
-            x = y.right
-        else:
-            x = y.left
-        x.parent = y.parent
-        if x.parent is self.root:
-            self.root.left = x
-        else:
-            if y is y.parent.left:
-                y.parent.left = x
-            else:
-                y.parent.right = x
-        if y is not z:
-            # y is the node to splice out, x is its child
-            y.left = z.left
-            y.right = z.right
-            y.parent = z.parent
-            z.left.parent = y
-            z.right.parent = y
-            if z is z.parent.left:
-                z.parent.left = y
-            else:
-                z.parent.right = y
-            if not y.red:
-                y.red = z.red
-                self.__delete_fixup(x)
-            else:
-                y.red = z.red
-        else:
-            if not y.red:
-                self.__delete_fixup(x)
-
-    cdef void __delete_fixup(self, RBNode x):
-        """Rebalance/fix RBTree after a deletion.  RBNode x is the
-        child of the spliced out node."""
-        cdef RBNode rootLeft = self.root.left
-        while not x.red and x is not rootLeft:
-            if x is x.parent.left:
-                w = x.parent.right
-                if w.red:
-                    w.red = False
-                    x.parent.red = True
-                    self.__rotate_left(x.parent)
-                    w = x.parent.right
-                if not w.right.red and not w.left.red:
-                    w.red = True
-                    x = x.parent
-                else:
-                    if not w.right.red:
-                        w.left.red = False
-                        w.red = True
-                        self.__rotate_right(w)
-                        w = x.parent.right
-                    w.red = x.parent.red
-                    x.parent.red = False
-                    w.right.red = False
-                    self.__rotate_left(x.parent)
-                    x = rootLeft # exit loop
-            else: # same as above, left/right switched
-                w = x.parent.left
-                if w.red:
-                    w.red = False
-                    x.parent.red = True
-                    self.__rotate_right(x.parent)
-                    w = x.parent.left
-                if not w.left.red and not w.right.red:
-                    w.red = True
-                    x = x.parent
-                else:
-                    if not w.left.red:
-                        w.right.red = False
-                        w.red = True
-                        self.__rotate_left(w)
-                        w = x.parent.left
-                    w.red = x.parent.red
-                    x.parent.red = False
-                    w.left.red = False
-                    self.__rotate_right(x.parent)
-                    x = rootLeft # exit loop
-        x.red = False
-
-    # Walking, searching
-    def __iter__(self):
-        return self.inorder()
-
-    def inorder(self, RBNode x = None):
-        """Generator that performs an inorder walk for the tree
-        rooted at RBNode x"""
-        if x is None:
-            x = self.getroot()
-        while x.left is not self.nil:
-            x = x.left
-        while x is not self.nil:
-            yield x
-            x = self.__successor(x)
-
-    cpdef RBNode find(self, double start, double end):
-        """Return the node with exactly the given start and end."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if start < x.start:
-                x = x.left
-            elif start == x.start:
-                if end == x.end:
-                    break # found it
-                elif end < x.end:
-                    x = x.left
-                else:
-                    x = x.right
-            else:
-                x = x.right
-        return x if x is not self.nil else None
-
-    cpdef RBNode find_left_end(self, double t):
-        """Find the leftmode node with end >= t.  With non-overlapping
-        intervals, this is the first node that might overlap time t.
-
-        Note that this relies on non-overlapping intervals, since
-        it assumes that we can use the endpoints to traverse the
-        tree even though it was created using the start points."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if t < x.end:
-                if x.left is self.nil:
-                    break
-                x = x.left
-            elif t == x.end:
-                break
-            else:
-                if x.right is self.nil:
-                    x = self.__successor(x)
-                    break
-                x = x.right
-        return x if x is not self.nil else None
-
-    cpdef RBNode find_right_start(self, double t):
-        """Find the rightmode node with start <= t.  With non-overlapping
-        intervals, this is the last node that might overlap time t."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if t < x.start:
-                if x.left is self.nil:
-                    x = self.__predecessor(x)
-                    break
-                x = x.left
-            elif t == x.start:
-                break
-            else:
-                if x.right is self.nil:
-                    break
-                x = x.right
-        return x if x is not self.nil else None
-
-    # Intersections
-    def intersect(self, double start, double end):
-        """Generator that returns nodes that overlap the given
-        (start,end) range.  Assumes non-overlapping intervals."""
-        # Start with the leftmode node that ends after start
-        cdef RBNode n = self.find_left_end(start)
-        while n is not None:
-            if n.start >= end:
-                # this node starts after the requested end; we're done
-                break
-            if start < n.end:
-                # this node overlaps our requested area
-                yield n
-            n = self.successor(n)

nilmdb/server/rbtree.pyxdep

@@ -1 +0,0 @@
-rbtree.pxd

nilmdb/server/rocket.c

@@ -1,806 +0,0 @@
-#include <Python.h>
-#include <structmember.h>
-#include <endian.h>
-
-#include <ctype.h>
-#include <stdint.h>
-
-#define __STDC_FORMAT_MACROS
-#include <inttypes.h>
-
-/* Values missing from stdint.h */
-#define UINT8_MIN 0
-#define UINT16_MIN 0
-#define UINT32_MIN 0
-#define UINT64_MIN 0
-
-/* Marker values (if min == max, skip range check) */
-#define FLOAT32_MIN 0
-#define FLOAT32_MAX 0
-#define FLOAT64_MIN 0
-#define FLOAT64_MAX 0
-
-typedef int64_t timestamp_t;
-
-/* Somewhat arbitrary, just so we can use fixed sizes for strings
-   etc. */
-static const int MAX_LAYOUT_COUNT = 1024;
-
-/* Error object and constants */
-static PyObject *ParseError;
-typedef enum {
-	ERR_OTHER,
-	ERR_NON_MONOTONIC,
-	ERR_OUT_OF_INTERVAL,
-} parseerror_code_t;
-static void add_parseerror_codes(PyObject *module)
-{
-	PyModule_AddIntMacro(module, ERR_OTHER);
-	PyModule_AddIntMacro(module, ERR_NON_MONOTONIC);
-	PyModule_AddIntMacro(module, ERR_OUT_OF_INTERVAL);
-}
-
-/* Helpers to raise ParseErrors.  Use "return raise_str(...)" etc. */
-static PyObject *raise_str(int line, int col, int code, const char *string)
-{
-	PyObject *o;
-	o = Py_BuildValue("(iiis)", line, col, code, string);
-	if (o != NULL) {
-		PyErr_SetObject(ParseError, o);
-		Py_DECREF(o);
-	}
-	return NULL;
-}
-static PyObject *raise_int(int line, int col, int code, int64_t num)
-{
-	PyObject *o;
-	o = Py_BuildValue("(iiiL)", line, col, code, (long long)num);
-	if (o != NULL) {
-		PyErr_SetObject(ParseError, o);
-		Py_DECREF(o);
-	}
-	return NULL;
-}
-
-/****
- * Layout and type helpers
- */
-typedef union {
-	int8_t i;
-	uint8_t u;
-} union8_t;
-typedef union {
-	int16_t i;
-	uint16_t u;
-} union16_t;
-typedef union {
-	int32_t i;
-	uint32_t u;
-	float f;
-} union32_t;
-typedef union {
-	int64_t i;
-	uint64_t u;
-	double d;
-} union64_t;
-
-typedef enum {
-	LAYOUT_TYPE_NONE,
-	LAYOUT_TYPE_INT8,
-	LAYOUT_TYPE_UINT8,
-	LAYOUT_TYPE_INT16,
-	LAYOUT_TYPE_UINT16,
-	LAYOUT_TYPE_INT32,
-	LAYOUT_TYPE_UINT32,
-	LAYOUT_TYPE_INT64,
-	LAYOUT_TYPE_UINT64,
-	LAYOUT_TYPE_FLOAT32,
-	LAYOUT_TYPE_FLOAT64,
-} layout_type_t;
-
-struct {
-	char *string;
-	layout_type_t layout;
-	int size;
-} type_lookup[] = {
-	{ "int8",    LAYOUT_TYPE_INT8,    1 },
-	{ "uint8",   LAYOUT_TYPE_UINT8,   1 },
-	{ "int16",   LAYOUT_TYPE_INT16,   2 },
-	{ "uint16",  LAYOUT_TYPE_UINT16,  2 },
-	{ "int32",   LAYOUT_TYPE_INT32,   4 },
-	{ "uint32",  LAYOUT_TYPE_UINT32,  4 },
-	{ "int64",   LAYOUT_TYPE_INT64,   8 },
-	{ "uint64",  LAYOUT_TYPE_UINT64,  8 },
-	{ "float32", LAYOUT_TYPE_FLOAT32, 4 },
-	{ "float64", LAYOUT_TYPE_FLOAT64, 8 },
-	{ NULL }
-};
-
-/****
- * Object definition, init, etc
- */
-
-/* Rocket object */
-typedef struct {
-	PyObject_HEAD
-	layout_type_t layout_type;
-	int layout_count;
-	int binary_size;
-	FILE *file;
-	int file_size;
-} Rocket;
-
-/* Dealloc / new */
-static void Rocket_dealloc(Rocket *self)
-{
-	if (self->file) {
-		fprintf(stderr, "rocket: file wasn't closed\n");
-		fclose(self->file);
-		self->file = NULL;
-	}
-	Py_TYPE(self)->tp_free((PyObject *)self);
-}
-
-static PyObject *Rocket_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
-	Rocket *self;
-
-	self = (Rocket *)type->tp_alloc(type, 0);
-	if (!self)
-		return NULL;
-	self->layout_type = LAYOUT_TYPE_NONE;
-	self->layout_count = 0;
-	self->binary_size = 0;
-	self->file = NULL;
-	self->file_size = -1;
-	return (PyObject *)self;
-}
-
-/* .__init__(layout, file) */
-static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
-{
-	const char *layout, *path;
-        int pathlen;
-	static char *kwlist[] = { "layout", "file", NULL };
-	if (!PyArg_ParseTupleAndKeywords(args, kwds, "sz#", kwlist,
-					 &layout, &path, &pathlen))
-		return -1;
-	if (!layout)
-		return -1;
-	if (path) {
-                if (strlen(path) != (size_t)pathlen) {
-                        PyErr_SetString(PyExc_ValueError, "path must not "
-                                        "contain NUL characters");
-                        return -1;
-                }
-		if ((self->file = fopen(path, "a+b")) == NULL) {
-			PyErr_SetFromErrno(PyExc_OSError);
-			return -1;
-		}
-		self->file_size = -1;
-	} else {
-		self->file = NULL;
-	}
-
-	const char *under;
-	char *tmp;
-	under = strchr(layout, '_');
-	if (!under) {
-		PyErr_SetString(PyExc_ValueError, "no such layout: "
-				"badly formatted string");
-		return -1;
-	}
-	self->layout_count = strtoul(under+1, &tmp, 10);
-	if (self->layout_count < 1 || *tmp != '\0') {
-		PyErr_SetString(PyExc_ValueError, "no such layout: "
-				"bad count");
-		return -1;
-	}
-	if (self->layout_count >= MAX_LAYOUT_COUNT) {
-		PyErr_SetString(PyExc_ValueError, "no such layout: "
-				"count too high");
-		return -1;
-	}
-
-	int i;
-	for (i = 0; type_lookup[i].string; i++)
-		if (strncmp(layout, type_lookup[i].string, under-layout) == 0)
-			break;
-	if (!type_lookup[i].string) {
-		PyErr_SetString(PyExc_ValueError, "no such layout: "
-				"bad data type");
-		return -1;
-	}
-	self->layout_type = type_lookup[i].layout;
-	self->binary_size = 8 + (type_lookup[i].size * self->layout_count);
-
-	return 0;
-}
-
-/* .close() */
-static PyObject *Rocket_close(Rocket *self)
-{
-	if (self->file) {
-		fclose(self->file);
-		self->file = NULL;
-	}
-	Py_INCREF(Py_None);
-	return Py_None;
-}
-
-/* .file_size property */
-static PyObject *Rocket_get_file_size(Rocket *self)
-{
-	if (!self->file) {
-		PyErr_SetString(PyExc_AttributeError, "no file");
-		return NULL;
-	}
-	if (self->file_size < 0) {
-		int oldpos;
-		if (((oldpos = ftell(self->file)) < 0) ||
-		    (fseek(self->file, 0, SEEK_END) < 0) ||
-		    ((self->file_size = ftell(self->file)) < 0) ||
-		    (fseek(self->file, oldpos, SEEK_SET) < 0)) {
-			PyErr_SetFromErrno(PyExc_OSError);
-			return NULL;
-		}
-	}
-	return PyLong_FromLong(self->file_size);
-}
-
-/****
- * Append from string
- */
-static inline long int strtoll10(const char *nptr, char **endptr) {
-	return strtoll(nptr, endptr, 10);
-}
-static inline long int strtoull10(const char *nptr, char **endptr) {
-	return strtoull(nptr, endptr, 10);
-}
-
-/* .append_string(count, data, offset, linenum, start, end, last_timestamp) */
-static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
-{
-	int count;
-	const char *data;
-	int offset;
-	const char *linestart;
-	int linenum;
-        long long ll1, ll2, ll3;
-	timestamp_t start;
-	timestamp_t end;
-	timestamp_t last_timestamp;
-
-	int written = 0;
-	char *endptr;
-	union8_t t8;
-	union16_t t16;
-	union32_t t32;
-	union64_t t64;
-	int i;
-
-        /* Input data is bytes.  Using 'y#' instead of 'y' might be
-           preferable, but strto* requires the null terminator. */
-	if (!PyArg_ParseTuple(args, "iyiiLLL:append_string", &count,
-			      &data, &offset, &linenum,
-			      &ll1, &ll2, &ll3))
-		return NULL;
-        start = ll1;
-        end = ll2;
-        last_timestamp = ll3;
-
-	/* Skip spaces, but don't skip over a newline. */
-#define SKIP_BLANK(buf) do {			\
-	while (isspace(*buf)) {			\
-		if (*buf == '\n')		\
-			break;			\
-		buf++;				\
-	} } while(0)
-
-	const char *buf = &data[offset];
-	while (written < count && *buf)
-	{
-		linestart = buf;
-		linenum++;
-
-		/* Skip leading whitespace and commented lines */
-		SKIP_BLANK(buf);
-		if (*buf == '#') {
-			while (*buf && *buf != '\n')
-				buf++;
-			if (*buf)
-				buf++;
-			continue;
-		}
-
-		/* Extract timestamp */
-		t64.i = strtoll(buf, &endptr, 10);
-		if (endptr == buf || !isspace(*endptr)) {
-			/* Try parsing as a double instead */
-			t64.d = strtod(buf, &endptr);
-			if (endptr == buf)
-				goto bad_timestamp;
-			if (!isspace(*endptr))
-				goto cant_parse_value;
-			t64.i = round(t64.d);
-		}
-		if (t64.i <= last_timestamp)
-			return raise_int(linenum, buf - linestart + 1,
-					 ERR_NON_MONOTONIC, t64.i);
-		last_timestamp = t64.i;
-		if (t64.i < start || t64.i >= end)
-			return raise_int(linenum, buf - linestart + 1,
-					 ERR_OUT_OF_INTERVAL, t64.i);
-		t64.u = le64toh(t64.u);
-		if (fwrite(&t64.u, 8, 1, self->file) != 1)
-			goto err;
-		buf = endptr;
-
-		/* Parse all values in the line */
-		switch (self->layout_type) {
-#define CS(type, parsefunc, parsetype, realtype, disktype, letoh, bytes) \
-		case LAYOUT_TYPE_##type:				\
-			/* parse and write in a loop */			\
-			for (i = 0; i < self->layout_count; i++) {	\
-				/* skip non-newlines */			\
-				SKIP_BLANK(buf);			\
-				if (*buf == '\n')			\
-					goto wrong_number_of_values;	\
-				/* parse number */			\
-				parsetype = parsefunc(buf, &endptr);	\
-				if (*endptr && !isspace(*endptr))	\
-					goto cant_parse_value;		\
-				/* check limits */			\
-				if (type##_MIN != type##_MAX &&		\
-				    (parsetype < type##_MIN ||		\
-				     parsetype > type##_MAX))		\
-					goto value_out_of_range;	\
-				/* convert to disk representation */	\
-				realtype = parsetype;			\
-				disktype = letoh(disktype);		\
-				/* write it */				\
-				if (fwrite(&disktype, bytes,		\
-					   1, self->file) != 1)		\
-					goto err;			\
-				/* advance buf */			\
-				buf = endptr;				\
-			}						\
-			/* Skip trailing whitespace and comments */	\
-			SKIP_BLANK(buf);				\
-			if (*buf == '#')				\
-				while (*buf && *buf != '\n')		\
-					buf++;				\
-			if (*buf == '\n')				\
-				buf++;					\
-			else if (*buf != '\0')				\
-				goto extra_data_on_line;		\
-			break
-
-			CS(INT8,   strtoll10,  t64.i, t8.i,  t8.u,         , 1);
-			CS(UINT8,  strtoull10, t64.u, t8.u,  t8.u,         , 1);
-			CS(INT16,  strtoll10,  t64.i, t16.i, t16.u, le16toh, 2);
-			CS(UINT16, strtoull10, t64.u, t16.u, t16.u, le16toh, 2);
-			CS(INT32,  strtoll10,  t64.i, t32.i, t32.u, le32toh, 4);
-			CS(UINT32, strtoull10, t64.u, t32.u, t32.u, le32toh, 4);
-			CS(INT64,  strtoll10,  t64.i, t64.i, t64.u, le64toh, 8);
-			CS(UINT64, strtoull10, t64.u, t64.u, t64.u, le64toh, 8);
-			CS(FLOAT32, strtod,   t64.d, t32.f, t32.u, le32toh, 4);
-			CS(FLOAT64, strtod,   t64.d, t64.d, t64.u, le64toh, 8);
-#undef CS
-		default:
-			PyErr_SetString(PyExc_TypeError, "unknown type");
-			return NULL;
-		}
-
-		/* Done this line */
-		written++;
-	}
-
-	fflush(self->file);
-
-	/* Build return value and return */
-	offset = buf - data;
-	PyObject *o;
-	o = Py_BuildValue("(iiLi)", written, offset,
-                          (long long)last_timestamp, linenum);
-	return o;
-err:
-	PyErr_SetFromErrno(PyExc_OSError);
-	return NULL;
-bad_timestamp:
-	return raise_str(linenum, buf - linestart + 1,
-			 ERR_OTHER, "bad timestamp");
-cant_parse_value:
-	return raise_str(linenum, buf - linestart + 1,
-			 ERR_OTHER, "can't parse value");
-wrong_number_of_values:
-	return raise_str(linenum, buf - linestart + 1,
-			 ERR_OTHER, "wrong number of values");
-value_out_of_range:
-	return raise_str(linenum, buf - linestart + 1,
-			 ERR_OTHER, "value out of range");
-extra_data_on_line:
-	return raise_str(linenum, buf - linestart + 1,
-			 ERR_OTHER, "extra data on line");
-}
-
-/****
- * Append from binary data
- */
-
-/* .append_binary(count, data, offset, linenum, start, end, last_timestamp) */
-static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
-{
-        int count;
-	const uint8_t *data;
-        int data_len;
-        int linenum;
-	int offset;
-        long long ll1, ll2, ll3;
-	timestamp_t start;
-	timestamp_t end;
-	timestamp_t last_timestamp;
-
-	if (!PyArg_ParseTuple(args, "iy#iiLLL:append_binary",
-                              &count, &data, &data_len, &offset,
-                              &linenum, &ll1, &ll2, &ll3))
-		return NULL;
-        start = ll1;
-        end = ll2;
-        last_timestamp = ll3;
-
-        /* Advance to offset */
-        if (offset > data_len)
-                return raise_str(0, 0, ERR_OTHER, "bad offset");
-        data += offset;
-        data_len -= offset;
-
-        /* Figure out max number of rows to insert */
-        int rows = data_len / self->binary_size;
-        if (rows > count)
-                rows = count;
-
-        /* Check timestamps */
-        timestamp_t ts;
-	int i;
-        for (i = 0; i < rows; i++) {
-                /* Read raw timestamp, byteswap if needed */
-                memcpy(&ts, &data[i * self->binary_size], 8);
-                ts = le64toh(ts);
-
-                /* Check limits */
-                if (ts <= last_timestamp)
-                        return raise_int(i, 0, ERR_NON_MONOTONIC, ts);
-                last_timestamp = ts;
-                if (ts < start || ts >= end)
-                        return raise_int(i, 0, ERR_OUT_OF_INTERVAL, ts);
-        }
-
-        /* Write binary data */
-        if (fwrite(data, self->binary_size, rows, self->file) != (size_t)rows) {
-                PyErr_SetFromErrno(PyExc_OSError);
-                return NULL;
-        }
-	fflush(self->file);
-
-	/* Build return value and return */
-	PyObject *o;
-	o = Py_BuildValue("(iiLi)", rows, offset + rows * self->binary_size,
-                          (long long)last_timestamp, linenum);
-	return o;
-}
-
-/****
- * Extract to binary bytes object containing ASCII text-formatted data
- */
-
-static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
-{
-	long count;
-	long offset;
-
-	if (!PyArg_ParseTuple(args, "ll", &offset, &count))
-		return NULL;
-	if (!self->file) {
-		PyErr_SetString(PyExc_Exception, "no file");
-		return NULL;
-	}
-	/* Seek to target location */
-	if (fseek(self->file, offset, SEEK_SET) < 0) {
-		PyErr_SetFromErrno(PyExc_OSError);
-		return NULL;
-	}
-
-	char *str = NULL, *new;
-	long len_alloc = 0;
-	long len = 0;
-	int ret;
-
-	/* min space free in string (and the maximum length of one
-	   line); this is generous */
-	const int min_free = 32 * MAX_LAYOUT_COUNT;
-
-	/* how much to allocate at once */
-	const int alloc_size = 1048576;
-
-	int row, i;
-	union8_t t8;
-	union16_t t16;
-	union32_t t32;
-	union64_t t64;
-	for (row = 0; row < count; row++) {
-		/* Make sure there's space for a line */
-		if ((len_alloc - len) < min_free) {
-			/* grow by 1 meg at a time */
-			len_alloc += alloc_size;
-			new = realloc(str, len_alloc);
-			if (new == NULL)
-				goto err;
-			str = new;
-		}
-
-		/* Read and print timestamp */
-		if (fread(&t64.u, 8, 1, self->file) != 1)
-			goto err;
-		t64.u = le64toh(t64.u);
-		ret = sprintf(&str[len], "%" PRId64, t64.i);
-		if (ret <= 0)
-			goto err;
-		len += ret;
-
-		/* Read and print values */
-		switch (self->layout_type) {
-#define CASE(type, fmt, fmttype, disktype, letoh, bytes)		\
-		case LAYOUT_TYPE_##type:				\
-			/* read and format in a loop */			\
-			for (i = 0; i < self->layout_count; i++) {	\
-				if (fread(&disktype, bytes,		\
-					  1, self->file) != 1)		\
-					goto err;			\
-				disktype = letoh(disktype);		\
-				ret = sprintf(&str[len], " " fmt,	\
-					      fmttype);			\
-				if (ret <= 0)				\
-					goto err;			\
-				len += ret;				\
-			}						\
-			break
-			CASE(INT8,   "%" PRId8,  t8.i,  t8.u,         , 1);
-			CASE(UINT8,  "%" PRIu8,  t8.u,  t8.u,         , 1);
-			CASE(INT16,  "%" PRId16, t16.i, t16.u, le16toh, 2);
-			CASE(UINT16, "%" PRIu16, t16.u, t16.u, le16toh, 2);
-			CASE(INT32,  "%" PRId32, t32.i, t32.u, le32toh, 4);
-			CASE(UINT32, "%" PRIu32, t32.u, t32.u, le32toh, 4);
-			CASE(INT64,  "%" PRId64, t64.i, t64.u, le64toh, 8);
-			CASE(UINT64, "%" PRIu64, t64.u, t64.u, le64toh, 8);
-			/* These next two are a bit debatable.  floats
-			   are 6-9 significant figures, so we print 7.
-			   Doubles are 15-19, so we print 17.  This is
-			   similar to the old prep format for float32.
-			*/
-			CASE(FLOAT32, "%.6e",  t32.f, t32.u, le32toh, 4);
-			CASE(FLOAT64, "%.16e", t64.d, t64.u, le64toh, 8);
-#undef CASE
-		default:
-			PyErr_SetString(PyExc_TypeError, "unknown type");
-			if (str) free(str);
-			return NULL;
-		}
-		str[len++] = '\n';
-	}
-
-	PyObject *pystr = PyBytes_FromStringAndSize(str, len);
-	free(str);
-	return pystr;
-err:
-	if (str) free(str);
-	PyErr_SetFromErrno(PyExc_OSError);
-	return NULL;
-}
-
-/****
- * Extract to binary bytes object containing raw little-endian binary data
- */
-static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args)
-{
-	long count;
-	long offset;
-
-	if (!PyArg_ParseTuple(args, "ll", &offset, &count))
-		return NULL;
-	if (!self->file) {
-		PyErr_SetString(PyExc_Exception, "no file");
-		return NULL;
-	}
-	/* Seek to target location */
-	if (fseek(self->file, offset, SEEK_SET) < 0) {
-		PyErr_SetFromErrno(PyExc_OSError);
-		return NULL;
-	}
-
-        uint8_t *str;
-        int len = count * self->binary_size;
-        str = malloc(len);
-        if (str == NULL) {
-                PyErr_SetFromErrno(PyExc_OSError);
-                return NULL;
-        }
-
-        /* Data in the file is already in the desired little-endian
-           binary format, so just read it directly. */
-        if (fread(str, self->binary_size, count, self->file) != (size_t)count) {
-                free(str);
-                PyErr_SetFromErrno(PyExc_OSError);
-                return NULL;
-        }
-
-	PyObject *pystr = PyBytes_FromStringAndSize((char *)str, len);
-	free(str);
-	return pystr;
-}
-
-/****
- * Extract timestamp
- */
-static PyObject *Rocket_extract_timestamp(Rocket *self, PyObject *args)
-{
-	long offset;
-	union64_t t64;
-	if (!PyArg_ParseTuple(args, "l", &offset))
-		return NULL;
-	if (!self->file) {
-		PyErr_SetString(PyExc_Exception, "no file");
-		return NULL;
-	}
-
-	/* Seek to target location and read timestamp */
-	if ((fseek(self->file, offset, SEEK_SET) < 0) ||
-	    (fread(&t64.u, 8, 1, self->file) != 1)) {
-		PyErr_SetFromErrno(PyExc_OSError);
-		return NULL;
-	}
-
-	/* Convert and return */
-	t64.u = le64toh(t64.u);
-	return Py_BuildValue("L", (long long)t64.i);
-}
-
-/****
- * Module and type setup
- */
-
-static PyGetSetDef Rocket_getsetters[] = {
-	{ "file_size", (getter)Rocket_get_file_size, NULL,
-	  "file size in bytes", NULL },
-	{ NULL },
-};
-
-static PyMemberDef Rocket_members[] = {
-	{ "binary_size", T_INT, offsetof(Rocket, binary_size), 0,
-	  "binary size per row" },
-	{ NULL },
-};
-
-static PyMethodDef Rocket_methods[] = {
-	{ "close",
-          (PyCFunction)Rocket_close, METH_NOARGS,
-	  "close(self)\n\n"
-	  "Close file handle" },
-
-	{ "append_string",
-          (PyCFunction)Rocket_append_string, METH_VARARGS,
-	  "append_string(self, count, data, offset, line, start, end, ts)\n\n"
-          "Parse string and append data.\n"
-	  "\n"
-	  "  count: maximum number of rows to add\n"
-          "  data: string data\n"
-          "  offset: byte offset into data to start parsing\n"
-          "  line: current line number of data\n"
-          "  start: starting timestamp for interval\n"
-          "  end: end timestamp for interval\n"
-          "  ts: last timestamp that was previously parsed\n"
-	  "\n"
-	  "Raises ParseError if timestamps are non-monotonic, outside\n"
-	  "the start/end interval etc.\n"
-	  "\n"
-          "On success, return a tuple:\n"
-          "  added_rows: how many rows were added from the file\n"
-          "  data_offset: current offset into the data string\n"
-          "  last_timestamp: last timestamp we parsed\n"
-          "  linenum: current line number" },
-
-	{ "append_binary",
-	  (PyCFunction)Rocket_append_binary, METH_VARARGS,
-	  "append_binary(self, count, data, offset, line, start, end, ts)\n\n"
-          "Append binary data, which must match the data layout.\n"
-	  "\n"
-	  "  count: maximum number of rows to add\n"
-          "  data: binary data\n"
-          "  offset: byte offset into data to start adding\n"
-          "  line: current line number (unused)\n"
-          "  start: starting timestamp for interval\n"
-          "  end: end timestamp for interval\n"
-          "  ts: last timestamp that was previously parsed\n"
-	  "\n"
-	  "Raises ParseError if timestamps are non-monotonic, outside\n"
-	  "the start/end interval etc.\n"
-	  "\n"
-          "On success, return a tuple:\n"
-          "  added_rows: how many rows were added from the file\n"
-          "  data_offset: current offset into the data string\n"
-          "  last_timestamp: last timestamp we parsed\n"
-          "  linenum: current line number (copied from argument)" },
-
-	{ "extract_string",
-          (PyCFunction)Rocket_extract_string, METH_VARARGS,
-	  "extract_string(self, offset, count)\n\n"
-	  "Extract count rows of data from the file at offset offset.\n"
-	  "Return an ascii formatted string according to the layout" },
-
-	{ "extract_binary",
-	  (PyCFunction)Rocket_extract_binary, METH_VARARGS,
-	  "extract_binary(self, offset, count)\n\n"
-	  "Extract count rows of data from the file at offset offset.\n"
-	  "Return a raw binary string of data matching the data layout." },
-
-	{ "extract_timestamp",
-	  (PyCFunction)Rocket_extract_timestamp, METH_VARARGS,
-	  "extract_timestamp(self, offset)\n\n"
-	  "Extract a single timestamp from the file" },
-
-	{ NULL },
-};
-
-static PyTypeObject RocketType = {
-	PyVarObject_HEAD_INIT(NULL, 0)
-
-	.tp_name	= "rocket.Rocket",
-	.tp_basicsize	= sizeof(Rocket),
-	.tp_flags	= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-
-	.tp_new		= Rocket_new,
-	.tp_dealloc	= (destructor)Rocket_dealloc,
-	.tp_init	= (initproc)Rocket_init,
-	.tp_methods	= Rocket_methods,
-	.tp_members	= Rocket_members,
-	.tp_getset	= Rocket_getsetters,
-
-	.tp_doc		= ("rocket.Rocket(layout, file)\n\n"
-			   "C implementation of the \"rocket\" data parsing\n"
-			   "interface, which translates between the binary\n"
-			   "format on disk and the ASCII or Python list\n"
-			   "format used when communicating with the rest of\n"
-			   "the system.")
-};
-
-static PyMethodDef module_methods[] = {
-	{ NULL },
-};
-
-static struct PyModuleDef moduledef = {
-        PyModuleDef_HEAD_INIT,
-        .m_name        = "rocker",
-        .m_doc         = "Rocket data parsing and formatting module",
-        .m_size        = -1,
-        .m_methods     = module_methods,
-};
-
-PyMODINIT_FUNC PyInit_rocket(void)
-{
-	PyObject *module;
-
-	RocketType.tp_new = PyType_GenericNew;
-	if (PyType_Ready(&RocketType) < 0)
-		return NULL;
-
-	module = PyModule_Create(&moduledef);
-	Py_INCREF(&RocketType);
-	PyModule_AddObject(module, "Rocket", (PyObject *)&RocketType);
-
-	ParseError = PyErr_NewException("rocket.ParseError", NULL, NULL);
-	Py_INCREF(ParseError);
-	PyModule_AddObject(module, "ParseError", ParseError);
-	add_parseerror_codes(module);
-
-	return module;
-}

nilmdb/server/server.py

@@ -1,546 +0,0 @@
-"""CherryPy-based server for accessing NILM database via HTTP"""
-
-import os
-import json
-import socket
-import traceback
-
-import psutil
-import cherrypy
-
-import nilmdb.server
-from nilmdb.utils.printf import sprintf
-from nilmdb.server.errors import NilmDBError
-from nilmdb.utils.time import string_to_timestamp
-
-from nilmdb.server.serverutil import (
-    chunked_response,
-    response_type,
-    exception_to_httperror,
-    CORS_allow,
-    json_to_request_params,
-    json_error_page,
-    cherrypy_start,
-    cherrypy_stop,
-    bool_param,
-    )
-
-# Add CORS_allow tool
-cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
-
-
-class NilmApp():
-    def __init__(self, db):
-        self.db = db
-
-
-# CherryPy apps
-class Root(NilmApp):
-    """Root application for NILM database"""
-    # /
-    @cherrypy.expose
-    def index(self):
-        cherrypy.response.headers['Content-Type'] = 'text/plain'
-        msg = sprintf("This is NilmDB version %s, running on host %s.\n",
-                      nilmdb.__version__, socket.getfqdn())
-        return msg
-
-    # /favicon.ico
-    @cherrypy.expose
-    def favicon_ico(self):
-        raise cherrypy.NotFound()
-
-    # /version
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def version(self):
-        return nilmdb.__version__
-
-    # /dbinfo
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def dbinfo(self):
-        """Return a dictionary with the database path,
-        size of the database in bytes, and free disk space in bytes"""
-        path = self.db.get_basepath()
-        usage = psutil.disk_usage(path)
-        dbsize = nilmdb.utils.du(path)
-        return {
-            "path": path,
-            "size": dbsize,
-            "other": max(usage.used - dbsize, 0),
-            "reserved": max(usage.total - usage.used - usage.free, 0),
-            "free": usage.free
-        }
-
-
-class Stream(NilmApp):
-    """Stream-specific operations"""
-
-    # Helpers
-    def _get_times(self, start_param, end_param):
-        (start, end) = (None, None)
-        try:
-            if start_param is not None:
-                start = string_to_timestamp(start_param)
-        except Exception:
-            raise cherrypy.HTTPError("400 Bad Request", sprintf(
-                "invalid start (%s): must be a numeric timestamp",
-                start_param))
-        try:
-            if end_param is not None:
-                end = string_to_timestamp(end_param)
-        except Exception:
-            raise cherrypy.HTTPError("400 Bad Request", sprintf(
-                "invalid end (%s): must be a numeric timestamp", end_param))
-        if start is not None and end is not None:
-            if start >= end:
-                raise cherrypy.HTTPError(
-                    "400 Bad Request",
-                    sprintf("start must precede end (%s >= %s)",
-                            start_param, end_param))
-        return (start, end)
-
-    # /stream/list
-    # /stream/list?layout=float32_8
-    # /stream/list?path=/newton/prep&extended=1
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def list(self, path=None, layout=None, extended=None):
-        """List all streams in the database.  With optional path or
-        layout parameter, just list streams that match the given path
-        or layout.
-
-        If extended is missing or zero, returns a list of lists
-        containing the path and layout: [ path, layout ]
-
-        If extended is true, returns a list of lists containing
-        extended info: [ path, layout, extent_min, extent_max,
-        total_rows, total_seconds ].  More data may be added.
-        """
-        return self.db.stream_list(path, layout, bool(extended))
-
-    # /stream/create?path=/newton/prep&layout=float32_8
-    @cherrypy.expose
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, ValueError)
-    @cherrypy.tools.CORS_allow(methods=["POST"])
-    def create(self, path, layout):
-        """Create a new stream in the database.  Provide path
-        and one of the nilmdb.layout.layouts keys.
-        """
-        return self.db.stream_create(path, layout)
-
-    # /stream/destroy?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError)
-    @cherrypy.tools.CORS_allow(methods=["POST"])
-    def destroy(self, path):
-        """Delete a stream.  Fails if any data is still present."""
-        return self.db.stream_destroy(path)
-
-    # /stream/rename?oldpath=/newton/prep&newpath=/newton/prep/1
-    @cherrypy.expose
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, ValueError)
-    @cherrypy.tools.CORS_allow(methods=["POST"])
-    def rename(self, oldpath, newpath):
-        """Rename a stream."""
-        return self.db.stream_rename(oldpath, newpath)
-
-    # /stream/get_metadata?path=/newton/prep
-    # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def get_metadata(self, path, key=None):
-        """Get metadata for the named stream.  If optional
-        key parameters are specified, only return metadata
-        matching the given keys."""
-        try:
-            data = self.db.stream_get_metadata(path)
-        except nilmdb.server.nilmdb.StreamError as e:
-            raise cherrypy.HTTPError("404 Not Found", str(e))
-        if key is None:  # If no keys specified, return them all
-            key = list(data.keys())
-        elif not isinstance(key, list):
-            key = [key]
-        result = {}
-        for k in key:
-            if k in data:
-                result[k] = data[k]
-            else:  # Return "None" for keys with no matching value
-                result[k] = None
-        return result
-
-    # Helper for set_metadata and get_metadata
-    def _metadata_helper(self, function, path, data):
-        if not isinstance(data, dict):
-            try:
-                data = dict(json.loads(data))
-            except TypeError as e:
-                raise NilmDBError("can't parse 'data' parameter: " + str(e))
-        for key in data:
-            if not isinstance(data[key], (str, float, int)):
-                raise NilmDBError("metadata values must be a string or number")
-        function(path, data)
-
-    # /stream/set_metadata?path=/newton/prep&data=<json>
-    @cherrypy.expose
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError)
-    @cherrypy.tools.CORS_allow(methods=["POST"])
-    def set_metadata(self, path, data):
-        """Set metadata for the named stream, replacing any existing
-        metadata.  Data can be json-encoded or a plain dictionary."""
-        self._metadata_helper(self.db.stream_set_metadata, path, data)
-
-    # /stream/update_metadata?path=/newton/prep&data=<json>
-    @cherrypy.expose
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError, ValueError)
-    @cherrypy.tools.CORS_allow(methods=["POST"])
-    def update_metadata(self, path, data):
-        """Set metadata for the named stream, replacing any existing
-        metadata.  Data can be json-encoded or a plain dictionary."""
-        self._metadata_helper(self.db.stream_update_metadata, path, data)
-
-    # /stream/insert?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, ValueError)
-    @cherrypy.tools.CORS_allow(methods=["PUT"])
-    def insert(self, path, start, end, binary=False):
-        """
-        Insert new data into the database.  Provide textual data
-        (matching the path's layout) as a HTTP PUT.
-
-        If 'binary' is True, expect raw binary data, rather than lines
-        of ASCII-formatted data.  Raw binary data is always
-        little-endian and matches the database types (including an
-        int64 timestamp).
-        """
-        binary = bool_param(binary)
-
-        # Important that we always read the input before throwing any
-        # errors, to keep lengths happy for persistent connections.
-        # Note that CherryPy 3.2.2 has a bug where this fails for GET
-        # requests, if we ever want to handle those (issue #1134)
-        body = cherrypy.request.body.read()
-
-        # Verify content type for binary data
-        content_type = cherrypy.request.headers.get('content-type')
-        if binary and content_type:
-            if content_type != "application/octet-stream":
-                raise cherrypy.HTTPError("400", "Content type must be "
-                                         "application/octet-stream for "
-                                         "binary data, not " + content_type)
-
-        # Note that non-binary data is *not* decoded from bytes to string,
-        # but rather passed directly to stream_insert.
-
-        # Check path and get layout
-        if len(self.db.stream_list(path=path)) != 1:
-            raise cherrypy.HTTPError("404", "No such stream: " + path)
-
-        # Check limits
-        (start, end) = self._get_times(start, end)
-
-        # Pass the data directly to nilmdb, which will parse it and
-        # raise a ValueError if there are any problems.
-        self.db.stream_insert(path, start, end, body, binary)
-
-        # Done
-        return
-
-    # /stream/remove?path=/newton/prep
-    # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    @cherrypy.expose
-    @cherrypy.tools.json_in()
-    @cherrypy.tools.CORS_allow(methods=["POST"])
-    @chunked_response
-    @response_type("application/x-json-stream")
-    def remove(self, path, start=None, end=None):
-        """
-        Remove data from the backend database.  Removes all data in
-        the interval [start, end).
-
-        Returns the number of data points removed.  Since this is a potentially
-        long-running operation, multiple numbers may be returned as the
-        data gets removed from the backend database.  The total number of
-        points removed is the sum of all of these numbers.
-        """
-        (start, end) = self._get_times(start, end)
-
-        if len(self.db.stream_list(path=path)) != 1:
-            raise cherrypy.HTTPError("404", "No such stream: " + path)
-
-        def content(start, end):
-            # Note: disable chunked responses to see tracebacks from here.
-            while True:
-                (removed, restart) = self.db.stream_remove(path, start, end)
-                response = json.dumps(removed) + "\r\n"
-                yield response.encode('utf-8')
-                if restart is None:
-                    break
-                start = restart
-        return content(start, end)
-
-    # /stream/intervals?path=/newton/prep
-    # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    # /stream/intervals?path=/newton/prep&diffpath=/newton/prep2
-    @cherrypy.expose
-    @chunked_response
-    @response_type("application/x-json-stream")
-    def intervals(self, path, start=None, end=None, diffpath=None):
-        """
-        Get intervals from backend database.  Streams the resulting
-        intervals as JSON strings separated by CR LF pairs.  This may
-        make multiple requests to the nilmdb backend to avoid causing
-        it to block for too long.
-
-        Returns intervals between 'start' and 'end' belonging to
-        'path'.  If 'diff' is provided, the set-difference between
-        intervals in 'path' and intervals in 'diffpath' are
-        returned instead.
-
-        Note that the response type is the non-standard
-        'application/x-json-stream' for lack of a better option.
-        """
-        (start, end) = self._get_times(start, end)
-
-        if len(self.db.stream_list(path=path)) != 1:
-            raise cherrypy.HTTPError("404", "No such stream: " + path)
-
-        if diffpath and len(self.db.stream_list(path=diffpath)) != 1:
-            raise cherrypy.HTTPError("404", "No such stream: " + diffpath)
-
-        def content(start, end):
-            # Note: disable chunked responses to see tracebacks from here.
-            while True:
-                (ints, restart) = self.db.stream_intervals(path, start, end,
-                                                           diffpath)
-                response = ''.join([json.dumps(i) + "\r\n" for i in ints])
-                yield response.encode('utf-8')
-                if restart is None:
-                    break
-                start = restart
-        return content(start, end)
-
-    # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
-    @cherrypy.expose
-    @chunked_response
-    def extract(self, path, start=None, end=None,
-                count=False, markup=False, binary=False):
-        """
-        Extract data from backend database.  Streams the resulting
-        entries as ASCII text lines separated by newlines.  This may
-        make multiple requests to the nilmdb backend to avoid causing
-        it to block for too long.
-
-        If 'count' is True, returns a count rather than actual data.
-
-        If 'markup' is True, adds comments to the stream denoting each
-        interval's start and end timestamp.
-
-        If 'binary' is True, return raw binary data, rather than lines
-        of ASCII-formatted data.  Raw binary data is always
-        little-endian and matches the database types (including an
-        int64 timestamp).
-        """
-        binary = bool_param(binary)
-        markup = bool_param(markup)
-        count = bool_param(count)
-
-        (start, end) = self._get_times(start, end)
-
-        # Check path and get layout
-        if len(self.db.stream_list(path=path)) != 1:
-            raise cherrypy.HTTPError("404", "No such stream: " + path)
-
-        if binary:
-            content_type = "application/octet-stream"
-            if markup or count:
-                raise cherrypy.HTTPError("400", "can't mix binary and "
-                                         "markup or count modes")
-        else:
-            content_type = "text/plain"
-        cherrypy.response.headers['Content-Type'] = content_type
-
-        def content(start, end):
-            # Note: disable chunked responses to see tracebacks from here.
-            if count:
-                matched = self.db.stream_extract(path, start, end,
-                                                 count=True)
-                yield sprintf(b"%d\n", matched)
-                return
-
-            while True:
-                (data, restart) = self.db.stream_extract(
-                    path, start, end, count=False,
-                    markup=markup, binary=binary)
-                yield data
-
-                if restart is None:
-                    return
-                start = restart
-        return content(start, end)
-
-
-class Exiter():
-    """App that exits the server, for testing"""
-    @cherrypy.expose
-    def index(self):
-        cherrypy.response.headers['Content-Type'] = 'text/plain'
-
-        def content():
-            yield b'Exiting by request'
-            raise SystemExit
-
-        return content()
-    index._cp_config = {'response.stream': True}
-
-
-class Server():
-    def __init__(self, db, host='127.0.0.1', port=8080,
-                 stoppable=False,        # whether /exit URL exists
-                 fast_shutdown=False,    # don't wait for clients to disconn.
-                 force_traceback=False,  # include traceback in all errors
-                 basepath='',            # base URL path for cherrypy.tree
-                 ):
-        # Save server version, just for verification during tests
-        self.version = nilmdb.__version__
-
-        self.db = db
-        if not getattr(db, "_thread_safe", None):
-            raise KeyError("Database object " + str(db) + " doesn't claim "
-                           "to be thread safe.  You should pass "
-                           "nilmdb.utils.serializer_proxy(NilmDB)(args) "
-                           "rather than NilmDB(args).")
-
-        # Build up global server configuration
-        cherrypy.config.update({
-            'environment': 'embedded',
-            'server.socket_host': host,
-            'server.socket_port': port,
-            'engine.autoreload.on': False,
-            'server.max_request_body_size': 8*1024*1024,
-            })
-
-        # Build up application specific configuration
-        app_config = {}
-        app_config.update({
-            'error_page.default': self.json_error_page,
-            })
-
-        # Some default headers to just help identify that things are working
-        app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
-
-        # Set up Cross-Origin Resource Sharing (CORS) handler so we
-        # can correctly respond to browsers' CORS preflight requests.
-        # This also limits verbs to GET and HEAD by default.
-        app_config.update({'tools.CORS_allow.on': True,
-                           'tools.CORS_allow.methods': ['GET', 'HEAD']})
-
-        # Configure the 'json_in' tool to also allow other content-types
-        # (like x-www-form-urlencoded), and to treat JSON as a dict that
-        # fills requests.param.
-        app_config.update({'tools.json_in.force': False,
-                           'tools.json_in.processor': json_to_request_params})
-
-        # Send tracebacks in error responses.  They're hidden by the
-        # error_page function for client errors (code 400-499).
-        app_config.update({'request.show_tracebacks': True})
-        self.force_traceback = force_traceback
-
-        # Patch CherryPy error handler to never pad out error messages.
-        # This isn't necessary, but then again, neither is padding the
-        # error messages.
-        cherrypy._cperror._ie_friendly_error_sizes = {}
-
-        # Build up the application and mount it
-        root = Root(self.db)
-        root.stream = Stream(self.db)
-        if stoppable:
-            root.exit = Exiter()
-        cherrypy.tree.apps = {}
-        cherrypy.tree.mount(root, basepath, config={"/": app_config})
-
-        # Shutdowns normally wait for clients to disconnect.  To speed
-        # up tests, set fast_shutdown = True
-        if fast_shutdown:
-            cherrypy.server.shutdown_timeout = 0
-        else:
-            cherrypy.server.shutdown_timeout = 5
-
-        # Set up the WSGI application pointer for external programs
-        self.wsgi_application = cherrypy.tree
-
-    def json_error_page(self, status, message, traceback, version):
-        """Return a custom error page in JSON so the client can parse it"""
-        return json_error_page(status, message, traceback, version,
-                               self.force_traceback)
-
-    def start(self, blocking=False, event=None):
-        cherrypy_start(blocking, event)
-
-    def stop(self):
-        cherrypy_stop()
-
-
-# Use a single global nilmdb.server.NilmDB and nilmdb.server.Server
-# instance since the database can only be opened once.  For this to
-# work, the web server must use only a single process and single
-# Python interpreter.  Multiple threads are OK.
-_wsgi_server = None
-
-
-def wsgi_application(dbpath, basepath):
-    """Return a WSGI application object with a database at the
-    specified path.
-
-    'dbpath' is a filesystem location, e.g. /home/nilm/db
-
-    'basepath' is the URL path of the application base, which
-    is the same as the first argument to Apache's WSGIScriptAlias
-    directive.
-    """
-    def application(environ, start_response):
-        global _wsgi_server
-        if _wsgi_server is None:
-            # Try to start the server
-            try:
-                db = nilmdb.utils.serializer_proxy(
-                    nilmdb.server.NilmDB)(dbpath)
-                _wsgi_server = nilmdb.server.Server(
-                    db, basepath=basepath.rstrip('/'))
-            except Exception:
-                # Build an error message on failure
-                import pprint
-                err = sprintf("Initializing database at path '%s' failed:\n\n",
-                              dbpath)
-                err += traceback.format_exc()
-                import pwd
-                import grp
-                err += sprintf("\nRunning as: uid=%d (%s), gid=%d (%s) "
-                               "on host %s, pid %d\n",
-                               os.getuid(), pwd.getpwuid(os.getuid())[0],
-                               os.getgid(), grp.getgrgid(os.getgid())[0],
-                               socket.gethostname(), os.getpid())
-                err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
-        if _wsgi_server is None:
-            # Serve up the error with our own mini WSGI app.
-            err_b = err.encode('utf-8')
-            headers = [('Content-type', 'text/plain; charset=utf-8'),
-                       ('Content-length', str(len(err_b)))]
-            start_response("500 Internal Server Error", headers)
-            return [err_b]
-
-        # Call the normal application
-        return _wsgi_server.wsgi_application(environ, start_response)
-    return application

nilmdb/server/serverutil.py

@@ -1,225 +0,0 @@
-"""Miscellaneous decorators and other helpers for running a CherryPy
-server"""
-
-import os
-import sys
-import json
-import decorator
-import functools
-import threading
-
-import cherrypy
-
-
-# Helper to parse parameters into booleans
-def bool_param(s):
-    """Return a bool indicating whether parameter 's' was True or False,
-    supporting a few different types for 's'."""
-    try:
-        ss = s.lower()
-        if ss in ["0", "false", "f", "no", "n"]:
-            return False
-        if ss in ["1", "true", "t", "yes", "y"]:
-            return True
-    except Exception:
-        return bool(s)
-    raise cherrypy.HTTPError("400 Bad Request",
-                             "can't parse parameter: " + ss)
-
-
-# Decorators
-def chunked_response(func):
-    """Decorator to enable chunked responses."""
-    # Set this to False to get better tracebacks from some requests
-    # (/stream/extract, /stream/intervals).
-    func._cp_config = {'response.stream': True}
-    return func
-
-
-def response_type(content_type):
-    """Return a decorator-generating function that sets the
-    response type to the specified string."""
-    def wrapper(func, *args, **kwargs):
-        cherrypy.response.headers['Content-Type'] = content_type
-        return func(*args, **kwargs)
-    return decorator.decorator(wrapper)
-
-
-def exception_to_httperror(*expected):
-    """Return a decorator-generating function that catches expected
-    errors and throws a HTTPError describing it instead.
-
-        @exception_to_httperror(NilmDBError, ValueError)
-        def foo():
-            pass
-    """
-    def wrapper(func, *args, **kwargs):
-        exc_info = None
-        try:
-            return func(*args, **kwargs)
-        except expected:
-            # Re-raise it, but maintain the original traceback
-            exc_info = sys.exc_info()
-            new_exc = cherrypy.HTTPError("400 Bad Request", str(exc_info[1]))
-            raise new_exc.with_traceback(exc_info[2])
-        finally:
-            del exc_info
-    # We need to preserve the function's argspecs for CherryPy to
-    # handle argument errors correctly.  Decorator.decorator takes
-    # care of that.
-    return decorator.decorator(wrapper)
-
-
-# Custom CherryPy tools
-def CORS_allow(methods):
-    """This does several things:
-
-    Handles CORS preflight requests.
-    Adds Allow: header to all requests.
-    Raise 405 if request.method not in method.
-
-    It is similar to cherrypy.tools.allow, with the CORS stuff added.
-
-    Add this to CherryPy with:
-    cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
-    """
-    request = cherrypy.request.headers
-    response = cherrypy.response.headers
-
-    if not isinstance(methods, (tuple, list)):
-        methods = [methods]
-    methods = [m.upper() for m in methods if m]
-    if not methods:
-        methods = ['GET', 'HEAD']
-    elif 'GET' in methods and 'HEAD' not in methods:
-        methods.append('HEAD')
-    response['Allow'] = ', '.join(methods)
-
-    # Allow all origins
-    if 'Origin' in request:
-        response['Access-Control-Allow-Origin'] = request['Origin']
-
-    # If it's a CORS request, send response.
-    request_method = request.get("Access-Control-Request-Method", None)
-    request_headers = request.get("Access-Control-Request-Headers", None)
-    if (cherrypy.request.method == "OPTIONS" and
-            request_method and request_headers):
-        response['Access-Control-Allow-Headers'] = request_headers
-        response['Access-Control-Allow-Methods'] = ', '.join(methods)
-        # Try to stop further processing and return a 200 OK
-        cherrypy.response.status = "200 OK"
-        cherrypy.response.body = b""
-        cherrypy.request.handler = lambda: ""
-        return
-
-    # Reject methods that were not explicitly allowed
-    if cherrypy.request.method not in methods:
-        raise cherrypy.HTTPError(405)
-
-
-# Helper for json_in tool to process JSON data into normal request
-# parameters.
-def json_to_request_params(body):
-    cherrypy.lib.jsontools.json_processor(body)
-    if not isinstance(cherrypy.request.json, dict):
-        raise cherrypy.HTTPError(415)
-    cherrypy.request.params.update(cherrypy.request.json)
-
-
-# Used as an "error_page.default" handler
-def json_error_page(status, message, traceback, version,
-                    force_traceback=False):
-    """Return a custom error page in JSON so the client can parse it"""
-    errordata = {"status": status,
-                 "message": message,
-                 "version": version,
-                 "traceback": traceback}
-    # Don't send a traceback if the error was 400-499 (client's fault)
-    code = int(status.split()[0])
-    if not force_traceback:
-        if 400 <= code <= 499:
-            errordata["traceback"] = ""
-    # Override the response type, which was previously set to text/html
-    cherrypy.serving.response.headers['Content-Type'] = (
-        "application/json;charset=utf-8")
-    # Undo the HTML escaping that cherrypy's get_error_page function applies
-    # (cherrypy issue 1135)
-    for k, v in errordata.items():
-        v = v.replace("&lt;", "<")
-        v = v.replace("&gt;", ">")
-        v = v.replace("&amp;", "&")
-        errordata[k] = v
-    return json.dumps(errordata, separators=(',', ':'))
-
-
-class CherryPyExit(SystemExit):
-    pass
-
-
-def cherrypy_patch_exit():
-    # Cherrypy stupidly calls os._exit(70) when it can't bind the port
-    # and exits.  Instead of that, raise a CherryPyExit (derived from
-    # SystemExit).  This exception may not make it back up to the caller
-    # due to internal thread use in the CherryPy engine, but there should
-    # be at least some indication that it happened.
-    bus = cherrypy.process.wspbus.bus
-    if "_patched_exit" in bus.__dict__:
-        return
-    bus._patched_exit = True
-
-    def patched_exit(orig):
-        real_exit = os._exit
-
-        def fake_exit(code):
-            raise CherryPyExit(code)
-        os._exit = fake_exit
-        try:
-            orig()
-        finally:
-            os._exit = real_exit
-    bus.exit = functools.partial(patched_exit, bus.exit)
-
-    # A behavior change in Python 3.8 means that some thread exceptions,
-    # derived from SystemExit, now print tracebacks where they didn't
-    # used to: https://bugs.python.org/issue1230540
-    # Install a thread exception hook that ignores CherryPyExit;
-    # to make this match the behavior where we didn't set
-    # threading.excepthook, we also need to ignore SystemExit.
-    def hook(args):
-        if args.exc_type == CherryPyExit or args.exc_type == SystemExit:
-            return
-        sys.excepthook(args.exc_type, args.exc_value,
-                       args.exc_traceback)  # pragma: no cover
-    threading.excepthook = hook
-
-
-# Start/stop CherryPy standalone server
-def cherrypy_start(blocking=False, event=False):
-    """Start the CherryPy server, handling errors and signals
-    somewhat gracefully."""
-
-    cherrypy_patch_exit()
-
-    # Start the server
-    cherrypy.engine.start()
-
-    # Signal that the engine has started successfully
-    if event is not None:
-        event.set()
-
-    if blocking:
-        try:
-            cherrypy.engine.wait(cherrypy.engine.states.EXITING,
-                                 interval=0.1, channel='main')
-        except (KeyboardInterrupt, IOError):
-            cherrypy.engine.log('Keyboard Interrupt: shutting down')
-            cherrypy.engine.exit()
-        except SystemExit:
-            cherrypy.engine.log('SystemExit raised: shutting down')
-            cherrypy.engine.exit()
-            raise
-
-
-# Stop CherryPy server
-def cherrypy_stop():
-    cherrypy.engine.exit()

nilmdb/test_fileinterval.py

@@ -0,0 +1,46 @@
+from nilmdb import Interval, IntervalSet, IntervalError, FileInterval
+from datetime import datetime
+from nose.tools import assert_raises
+
+from test_interval import iset
+
+def fiset(string):
+    """Like iset, but builds with FileIntervals instead of Intervals"""
+    iset = IntervalSet()
+    for i, c in enumerate(string):
+        day = datetime.strptime("{0:04d}".format(i+2000), "%Y")
+        if (c == "["):
+            start = day
+        elif (c == "|"):
+            iset += FileInterval(start, day, "test.dat")
+            start = day
+        elif (c == "]"):
+            iset += FileInterval(start, day, "test.dat")
+            del start
+    return iset
+
+def test_fileinterval_vs_interval():
+    """Test FileInterval/Interval inheritance"""
+
+    i = iset("[--]")
+    f = fiset("[--]")
+
+    # check types
+    assert(isinstance(i[0], Interval))
+    assert(not isinstance(i[0], FileInterval))
+    assert(isinstance(f[0], Interval))
+    assert(isinstance(f[0], FileInterval))
+
+    # when doing an intersection, result should be a subset of the first arg
+    u = (i & f)
+    assert(isinstance(u[0], Interval))
+    assert(not isinstance(u[0], FileInterval))
+    u = (f & i)
+    assert(isinstance(u[0], Interval))
+    assert(isinstance(u[0], FileInterval))
+
+    # they're still the same though
+    assert(i == f == u)
+
+    # just for coverage
+    assert_raises(IntervalError, fiset("[]")[0].subset, f[0].start, f[0].end)

nilmdb/test_interval.py

@@ -0,0 +1,189 @@
+from nilmdb import Interval, IntervalSet, IntervalError
+from datetime import datetime
+from nose.tools import assert_raises
+import itertools 
+
+def test_interval():
+    """Test the Interval class"""
+    d1 = datetime.strptime("19801205","%Y%m%d")
+    d2 = datetime.strptime("19900216","%Y%m%d")
+    d3 = datetime.strptime("20111205","%Y%m%d")
+
+    # basic construction
+    i = Interval(d1, d1)
+    i = Interval(d1, d3)
+    assert(i.start == d1)
+    assert(i.end == d3)
+
+    # assignment should work
+    i.start = d2
+    try:
+        i.end = d1
+        raise Exception("should have died there")
+    except IntervalError:
+        pass
+    i.start = d1
+    i.end = d2
+
+    # end before start
+    assert_raises(IntervalError, Interval, d3, d1)
+
+    # wrong type
+    assert_raises(IntervalError, Interval, 1, 2)
+
+    # compare
+    assert(Interval(d1, d2) == Interval(d1, d2))
+    assert(Interval(d1, d2) < Interval(d1, d3))
+    assert(Interval(d1, d3) > Interval(d1, d2))
+    assert(Interval(d1, d2) < Interval(d2, d3))
+    assert(Interval(d1, d3) < Interval(d2, d3))
+    assert(Interval(d2, d2) > Interval(d1, d3))
+    assert(Interval(d3, d3) == Interval(d3, d3))
+    assert_raises(TypeError, cmp, i, 123)
+
+    # subset
+    assert(Interval(d1, d3).subset(d1, d2) == Interval(d1, d2))
+    assert_raises(IntervalError, Interval(d2, d3).subset, d1, d2)
+
+    # append
+    assert(Interval(d1, d2).is_adjacent(Interval(d2,d3)))
+    assert(Interval(d2, d3).is_adjacent(Interval(d1,d2)))
+    assert(not Interval(d2, d3).is_adjacent(Interval(d1,d3)))
+    assert_raises(TypeError, Interval(d1, d2).is_adjacent, 1)
+
+    # misc
+    assert(repr(i) == repr(eval(repr(i).replace("datetime.",""))))
+    assert(str(i) == "[1980-12-05 00:00:00 -> 1990-02-16 00:00:00]")
+
+def test_interval_intersect():
+    """Test Interval intersections"""
+    dates = [ datetime.strptime(year, "%y") for year in [ "00", "01", "02", "03" ] ]
+    perm = list(itertools.permutations(dates, 2))
+    prod = list(itertools.product(perm, perm))
+    should_intersect = {
+        False: [4, 5, 8, 20, 48, 56, 60, 96, 97, 100],
+        True: [0, 1, 2, 12, 13, 14, 16, 17, 24, 25, 26, 28, 29,
+               32, 49, 50, 52, 53, 61, 62, 64, 65, 68, 98, 101, 104]}
+    for i,((a,b),(c,d)) in enumerate(prod):
+        try:
+            i1 = Interval(a, b)
+            i2 = Interval(c, d)
+            assert(i1.intersects(i2) == i2.intersects(i1))
+            assert(i in should_intersect[i1.intersects(i2)])
+        except IntervalError:
+            assert(i not in should_intersect[True] and
+                   i not in should_intersect[False])
+    assert_raises(TypeError, i1.intersects, 1234)
+
+def test_intervalset_construct():
+    """Test interval set construction"""
+    dates = [ datetime.strptime(year, "%y") for year in [ "00", "01", "02", "03" ]]
+
+    a = Interval(dates[0], dates[1])
+    b = Interval(dates[1], dates[2])
+    c = Interval(dates[0], dates[2])
+    d = Interval(dates[2], dates[3])
+
+    iseta = IntervalSet(a)
+    isetb = IntervalSet([a, b])
+    isetc = IntervalSet([a])
+    assert(iseta != isetb)
+    assert(iseta == isetc)
+    assert(iseta != 3)
+    assert(IntervalSet(a) != IntervalSet(b))
+
+    print iseta == None
+    assert_raises(TypeError, cmp, iseta, isetb)
+    assert_raises(IntervalError, IntervalSet, [a, b, c])
+    assert_raises(TypeError, IntervalSet, [1, 2])
+
+    iset = IntervalSet(isetb)   # test iterator
+    assert(iset == isetb)
+    assert(len(iset) == 2)
+    assert(len(IntervalSet()) == 0)
+
+    # Test adding
+    iset = IntervalSet(a)
+    iset += IntervalSet(b)
+    assert(iset == IntervalSet([a, b]))
+    iset = IntervalSet(a)
+    iset += b
+    assert(iset == IntervalSet([a, b]))
+    iset = IntervalSet(a) + IntervalSet(b)
+    assert(iset == IntervalSet([a, b]))
+    iset = IntervalSet(b) + a
+    assert(iset == IntervalSet([a, b]))
+
+    # A set consisting of [0-1],[1-2] should match a set consisting of [0-2]
+    assert(IntervalSet([a,b]) == IntervalSet([c]))
+    # Etc
+    assert(IntervalSet([a,d]) != IntervalSet([c]))
+    assert(IntervalSet([c]) != IntervalSet([a,d]))
+    assert(IntervalSet([c,d]) != IntervalSet([b,d]))
+    
+    # misc
+    assert(repr(iset) == repr(eval(repr(iset).replace("datetime.",""))))
+
+def iset(string):
+    """Build an IntervalSet from a string, for testing purposes
+
+    Each character is a year
+    [ = interval start
+    | = interval end + adjacent start
+    ] = interval end
+    anything else is ignored
+    """
+    iset = IntervalSet()
+    for i, c in enumerate(string):
+        day = datetime.strptime("{0:04d}".format(i+2000), "%Y")
+        if (c == "["):
+            start = day
+        elif (c == "|"):
+            iset += Interval(start, day)
+            start = day
+        elif (c == "]"):
+            iset += Interval(start, day)
+            del start
+    return iset
+
+def test_intervalset_iset():
+    """Test basic iset construction"""
+    assert(iset("  [----]   ") ==
+           iset("  [-|--]   "))
+
+    assert(iset("[]  [--]   ") +
+           iset(" []    [--]") ==
+           iset("[|] [-----]"))
+
+def test_intervalset_intsersect():
+    """Test intersection (&)"""
+    assert_raises(TypeError, iset("[--]").__and__, 1234)
+    
+    assert(iset("[---------]") &
+           iset(" [---]     ") ==
+           iset(" [---]     "))
+
+    assert(iset(" [---]     ") &
+           iset("[---------]") ==
+           iset(" [---]     "))
+
+    assert(iset("    [-----]") &
+           iset(" [-----]   ") ==
+           iset("    [--]   "))
+
+    assert(iset("      [---]") &
+           iset(" [--]      ") ==
+           iset("           "))
+
+    assert(iset("    [-|---]") &
+           iset(" [-----|-] ") ==
+           iset("    [----] "))
+
+    assert(iset("    [-|-]  ") &
+           iset(" [-|--|--] ") ==
+           iset("    [---]  "))
+
+    assert(iset(" [----][--]") &
+           iset("[-] [--] []") ==
+           iset(" [] [-]  []"))
+

nilmdb/utils/__init__.py

@@ -1,16 +0,0 @@
-"""NilmDB utilities"""
-
-
-from nilmdb.utils.timer import Timer
-from nilmdb.utils.serializer import serializer_proxy
-from nilmdb.utils.lrucache import lru_cache
-from nilmdb.utils.diskusage import du, human_size
-from nilmdb.utils.mustclose import must_close
-from nilmdb.utils import atomic
-import nilmdb.utils.threadsafety
-import nilmdb.utils.fallocate
-import nilmdb.utils.time
-import nilmdb.utils.iterator
-import nilmdb.utils.interval
-import nilmdb.utils.lock
-import nilmdb.utils.sort

nilmdb/utils/atomic.py

@@ -1,19 +0,0 @@
-# Atomic file writing helper.
-
-import os
-
-
-def replace_file(filename, content):
-    """Attempt to atomically and durably replace the filename with the
-    given contents"""
-
-    newfilename = filename + b".new"
-
-    # Write to new file, flush it
-    with open(newfilename, "wb") as f:
-        f.write(content)
-        f.flush()
-        os.fsync(f.fileno())
-
-    # Move new file over old one
-    os.replace(newfilename, filename)

nilmdb/utils/diskusage.py

@@ -1,36 +0,0 @@
-import os
-import errno
-from math import log
-
-
-def human_size(num):
-    """Human friendly file size"""
-    unit_list = list(zip(['bytes', 'kiB', 'MiB', 'GiB', 'TiB'],
-                         [0, 0, 1, 2, 2]))
-    if num == 0:
-        return '0 bytes'
-    if num == 1:
-        return '1 byte'
-    exponent = min(int(log(num, 1024)), len(unit_list) - 1)
-    quotient = float(num) / 1024**exponent
-    unit, num_decimals = unit_list[exponent]
-    format_string = '{:.%sf} {}' % (num_decimals)
-    return format_string.format(quotient, unit)
-
-
-def du(path):
-    """Like du -sb, returns total size of path in bytes.  Ignore
-    errors that might occur if we encounter broken symlinks or
-    files in the process of being removed."""
-    try:
-        st = os.stat(path)
-        size = st.st_blocks * 512
-        if os.path.isdir(path):
-            for thisfile in os.listdir(path):
-                filepath = os.path.join(path, thisfile)
-                size += du(filepath)
-        return size
-    except OSError as e:
-        if e.errno != errno.ENOENT:
-            raise
-        return 0

nilmdb/utils/fallocate.py

@@ -1,20 +0,0 @@
-# Implementation of hole punching via fallocate, if the OS
-# and filesystem support it.
-
-import fallocate
-
-
-def punch_hole(filename, offset, length, ignore_errors=True):
-    """Punch a hole in the file.  This isn't well supported, so errors
-    are ignored by default."""
-    try:
-        with open(filename, "r+") as f:
-            fallocate.fallocate(
-                f.fileno(),
-                offset,
-                length,
-                fallocate.FALLOC_FL_KEEP_SIZE | fallocate.FALLOC_FL_PUNCH_HOLE)
-    except Exception:
-        if ignore_errors:
-            return
-        raise

nilmdb/utils/interval.py

@@ -1,168 +0,0 @@
-"""Interval.  Like nilmdb.server.interval, but re-implemented here
-in plain Python so clients have easier access to it, and with a few
-helper functions.
-
-Intervals are half-open, ie. they include data points with timestamps
-[start, end)
-"""
-
-import nilmdb.utils.time
-import nilmdb.utils.iterator
-
-
-class IntervalError(Exception):
-    """Error due to interval overlap, etc"""
-    pass
-
-
-# Interval
-class Interval:
-    """Represents an interval of time."""
-
-    def __init__(self, start, end):
-        """
-        'start' and 'end' are arbitrary numbers that represent time
-        """
-        if start >= end:
-            # Explicitly disallow zero-width intervals, since they're half-open
-            raise IntervalError("start %s must precede end %s" % (start, end))
-        self.start = start
-        self.end = end
-
-    def __repr__(self):
-        s = repr(self.start) + ", " + repr(self.end)
-        return self.__class__.__name__ + "(" + s + ")"
-
-    def __str__(self):
-        return ("[" + nilmdb.utils.time.timestamp_to_string(self.start) +
-                " -> " + nilmdb.utils.time.timestamp_to_string(self.end) + ")")
-
-    def human_string(self):
-        return ("[ " + nilmdb.utils.time.timestamp_to_human(self.start) +
-                " -> " + nilmdb.utils.time.timestamp_to_human(self.end) + " ]")
-
-    # Compare two intervals.  If non-equal, order by start then end
-    def __lt__(self, other):
-        return (self.start, self.end) < (other.start, other.end)
-
-    def __gt__(self, other):
-        return (self.start, self.end) > (other.start, other.end)
-
-    def __le__(self, other):
-        return (self.start, self.end) <= (other.start, other.end)
-
-    def __ge__(self, other):
-        return (self.start, self.end) >= (other.start, other.end)
-
-    def __eq__(self, other):
-        return (self.start, self.end) == (other.start, other.end)
-
-    def __ne__(self, other):
-        return (self.start, self.end) != (other.start, other.end)
-
-    def intersects(self, other):
-        """Return True if two Interval objects intersect"""
-        if not isinstance(other, Interval):
-            raise TypeError("need an Interval")
-        if self.end <= other.start or self.start >= other.end:
-            return False
-        return True
-
-    def subset(self, start, end):
-        """Return a new Interval that is a subset of this one"""
-        # A subclass that tracks additional data might override this.
-        if start < self.start or end > self.end:
-            raise IntervalError("not a subset")
-        return Interval(start, end)
-
-
-def _interval_math_helper(a, b, op, subset=True):
-    """Helper for set_difference, intersection functions,
-    to compute interval subsets based on a math operator on ranges
-    present in A and B.  Subsets are computed from A, or new intervals
-    are generated if subset = False."""
-    # Iterate through all starts and ends in sorted order.  Add a
-    # tag to the iterator so that we can figure out which one they
-    # were, after sorting.
-    def decorate(it, key_start, key_end):
-        for i in it:
-            yield i.start, key_start, i
-            yield i.end, key_end, i
-    a_iter = decorate(iter(a), 0, 2)
-    b_iter = decorate(iter(b), 1, 3)
-
-    # Now iterate over the timestamps of each start and end.
-    # At each point, evaluate which type of end it is, to determine
-    # how to build up the output intervals.
-    a_interval = None
-    in_a = False
-    in_b = False
-    out_start = None
-    for (ts, k, i) in nilmdb.utils.iterator.imerge(a_iter, b_iter):
-        if k == 0:
-            a_interval = i
-            in_a = True
-        elif k == 1:
-            in_b = True
-        elif k == 2:
-            in_a = False
-        else:  # k == 3
-            in_b = False
-        include = op(in_a, in_b)
-        if include and out_start is None:
-            out_start = ts
-        elif not include:
-            if out_start is not None and out_start != ts:
-                if subset:
-                    yield a_interval.subset(out_start, ts)
-                else:
-                    yield Interval(out_start, ts)
-            out_start = None
-
-
-def set_difference(a, b):
-    """
-    Compute the difference (a \\ b) between the intervals in 'a' and
-    the intervals in 'b'; i.e., the ranges that are present in 'self'
-    but not 'other'.
-
-    'a' and 'b' must both be iterables.
-
-    Returns a generator that yields each interval in turn.
-    Output intervals are built as subsets of the intervals in the
-    first argument (a).
-    """
-    return _interval_math_helper(a, b, (lambda a, b: a and not b))
-
-
-def intersection(a, b):
-    """
-    Compute the intersection between the intervals in 'a' and the
-    intervals in 'b'; i.e., the ranges that are present in both 'a'
-    and 'b'.
-
-    'a' and 'b' must both be iterables.
-
-    Returns a generator that yields each interval in turn.
-    Output intervals are built as subsets of the intervals in the
-    first argument (a).
-    """
-    return _interval_math_helper(a, b, (lambda a, b: a and b))
-
-
-def optimize(it):
-    """
-    Given an iterable 'it' with intervals, optimize them by joining
-    together intervals that are adjacent in time, and return a generator
-    that yields the new intervals.
-    """
-    saved_int = None
-    for interval in it:
-        if saved_int is not None:
-            if saved_int.end == interval.start:
-                interval.start = saved_int.start
-            else:
-                yield saved_int
-        saved_int = interval
-    if saved_int is not None:
-        yield saved_int

nilmdb/utils/iterator.py

@@ -1,38 +0,0 @@
-# Misc iterator tools
-
-# Iterator merging, based on http://code.activestate.com/recipes/491285/
-import heapq
-
-
-def imerge(*iterables):
-    '''Merge multiple sorted inputs into a single sorted output.
-
-    Equivalent to:  sorted(itertools.chain(*iterables))
-
-    >>> list(imerge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
-    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
-
-    '''
-    heappop, siftup, _Stop = heapq.heappop, heapq._siftup, StopIteration
-
-    h = []
-    h_append = h.append
-    for it in map(iter, iterables):
-        try:
-            nexter = it.__next__
-            h_append([nexter(), nexter])
-        except _Stop:
-            pass
-    heapq.heapify(h)
-
-    while 1:
-        try:
-            while 1:
-                v, nexter = s = h[0]    # raises IndexError when h is empty
-                yield v
-                s[0] = nexter()         # raises StopIteration when exhausted
-                siftup(h, 0)            # restore heap condition
-        except _Stop:
-            heappop(h)                  # remove empty iterator
-        except IndexError:
-            return

nilmdb/utils/lock.py

@@ -1,22 +0,0 @@
-# File locking
-
-import fcntl
-import errno
-
-
-def exclusive_lock(f):
-    """Acquire an exclusive lock.  Returns True on successful
-    lock, or False on error."""
-    try:
-        fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
-    except IOError as e:
-        if e.errno in (errno.EACCES, errno.EAGAIN):
-            return False
-        else:
-            raise
-    return True
-
-
-def exclusive_unlock(f):
-    """Release an exclusive lock."""
-    fcntl.flock(f.fileno(), fcntl.LOCK_UN)

nilmdb/utils/lrucache.py

@@ -1,80 +0,0 @@
-# Memoize a function's return value with a least-recently-used cache
-# Based on:
-#   http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators/
-# with added 'destructor' functionality.
-
-import collections
-import decorator
-
-
-def lru_cache(size=10, onremove=None, keys=slice(None)):
-    """Least-recently-used cache decorator.
-
-    @lru_cache(size=10, onremove=None)
-    def f(...):
-        pass
-
-    Given a function and arguments, memoize its return value.  Up to
-    'size' elements are cached.  'keys' is a slice object that
-    represents which arguments are used as the cache key.
-
-    When evicting a value from the cache, call the function
-    'onremove' with the value that's being evicted.
-
-    Call f.cache_remove(...) to evict the cache entry with the given
-    arguments.  Call f.cache_remove_all() to evict all entries.
-    f.cache_hits and f.cache_misses give statistics.
-    """
-
-    def decorate(func):
-        cache = collections.OrderedDict()  # order: least- to most-recent
-
-        def evict(value):
-            if onremove:
-                onremove(value)
-
-        def wrapper(orig, *args, **kwargs):
-            if kwargs:
-                raise NotImplementedError("kwargs not supported")
-            key = args[keys]
-            try:
-                value = cache.pop(key)
-                orig.cache_hits += 1
-            except KeyError:
-                value = orig(*args)
-                orig.cache_misses += 1
-                if len(cache) >= size:
-                    evict(cache.popitem(0)[1])  # evict LRU cache entry
-            cache[key] = value                  # (re-)insert this key at end
-            return value
-
-        def cache_remove(*args):
-            """Remove the described key from this cache, if present."""
-            key = args
-            if key in cache:
-                evict(cache.pop(key))
-            else:
-                if cache:
-                    if len(args) != len(next(iter(cache.keys()))):
-                        raise KeyError("trying to remove from LRU cache, but "
-                                       "number of arguments doesn't match the "
-                                       "cache key length")
-
-        def cache_remove_all():
-            nonlocal cache
-            for key in cache:
-                evict(cache[key])
-            cache = collections.OrderedDict()
-
-        def cache_info():
-            return (func.cache_hits, func.cache_misses)
-
-        new = decorator.decorator(wrapper, func)
-        func.cache_hits = 0
-        func.cache_misses = 0
-        new.cache_info = cache_info
-        new.cache_remove = cache_remove
-        new.cache_remove_all = cache_remove_all
-        return new
-
-    return decorate

nilmdb/utils/mustclose.py

@@ -1,71 +0,0 @@
-import sys
-import inspect
-import decorator
-from nilmdb.utils.printf import fprintf
-
-
-def must_close(errorfile=sys.stderr, wrap_verify=False):
-    """Class decorator that warns on 'errorfile' at deletion time if
-    the class's close() member wasn't called.
-
-    If 'wrap_verify' is True, every class method is wrapped with a
-    verifier that will raise AssertionError if the .close() method has
-    already been called."""
-    def class_decorator(cls):
-
-        def is_method_or_function(x):
-            return inspect.ismethod(x) or inspect.isfunction(x)
-
-        def wrap_class_method(wrapper):
-            try:
-                orig = getattr(cls, wrapper.__name__)
-            except AttributeError:
-                orig = lambda x: None
-            if is_method_or_function(orig):
-                setattr(cls, wrapper.__name__,
-                        decorator.decorator(wrapper, orig))
-
-        @wrap_class_method
-        def __init__(orig, self, *args, **kwargs):
-            ret = orig(self, *args, **kwargs)
-            self.__dict__["_must_close"] = True
-            self.__dict__["_must_close_initialized"] = True
-            return ret
-
-        @wrap_class_method
-        def __del__(orig, self, *args, **kwargs):
-            try:
-                if "_must_close" in self.__dict__:
-                    fprintf(errorfile, "error: %s.close() wasn't called!\n",
-                            self.__class__.__name__)
-                return orig(self, *args, **kwargs)
-            except:
-                pass
-
-        @wrap_class_method
-        def close(orig, self, *args, **kwargs):
-            if "_must_close" in self.__dict__:
-                del self._must_close
-            return orig(self, *args, **kwargs)
-
-        # Optionally wrap all other functions
-        def verifier(orig, self, *args, **kwargs):
-            if ("_must_close" not in self.__dict__ and
-                    "_must_close_initialized" in self.__dict__):
-                raise AssertionError("called " + str(orig) + " after close")
-            return orig(self, *args, **kwargs)
-        if wrap_verify:
-            for (name, method) in inspect.getmembers(cls,
-                                                     is_method_or_function):
-                # Skip some methods
-                if name in ["__del__", "__init__"]:
-                    continue
-                # Set up wrapper
-                if inspect.ismethod(method):
-                    func = method.__func__
-                else:
-                    func = method
-                setattr(cls, name, decorator.decorator(verifier, func))
-
-        return cls
-    return class_decorator

nilmdb/utils/printf.py

@@ -1,13 +0,0 @@
-"""printf, fprintf, sprintf"""
-
-
-def printf(_str, *args):
-    print(_str % args, end='')
-
-
-def fprintf(_file, _str, *args):
-    print(_str % args, end='', file=_file)
-
-
-def sprintf(_str, *args):
-    return (_str % args)

nilmdb/utils/serializer.py

@@ -1,134 +0,0 @@
-import queue
-import threading
-import sys
-
-# This file provides a class that will wrap an object and serialize
-# all calls to its methods.  All calls to that object will be queued
-# and executed from a single thread, regardless of which thread makes
-# the call.
-
-# Based partially on http://stackoverflow.com/questions/2642515/
-
-
-class SerializerThread(threading.Thread):
-    """Thread that retrieves call information from the queue, makes the
-    call, and returns the results."""
-    def __init__(self, classname, call_queue):
-        threading.Thread.__init__(self)
-        self.name = "Serializer-" + classname + "-" + self.name
-        self.call_queue = call_queue
-
-    def run(self):
-        while True:
-            result_queue, func, args, kwargs = self.call_queue.get()
-            # Terminate if result_queue is None
-            if result_queue is None:
-                return
-            exception = None
-            result = None
-            try:
-                result = func(*args, **kwargs)  # wrapped
-            except:
-                exception = sys.exc_info()
-            # Ensure we delete these before returning a result, so
-            # we don't unncessarily hold onto a reference while
-            # we're waiting for the next call.
-            del func, args, kwargs
-            result_queue.put((exception, result))
-            del exception, result
-
-
-def serializer_proxy(obj_or_type):
-    """Wrap the given object or type in a SerializerObjectProxy.
-
-    Returns a SerializerObjectProxy object that proxies all method
-    calls to the object, as well as attribute retrievals.
-
-    The proxied requests, including instantiation, are performed in a
-    single thread and serialized between caller threads.
-    """
-    class SerializerCallProxy():
-        def __init__(self, call_queue, func, objectproxy):
-            self.call_queue = call_queue
-            self.func = func
-            # Need to hold a reference to object proxy so it doesn't
-            # go away (and kill the thread) until after get called.
-            self.objectproxy = objectproxy
-
-        def __call__(self, *args, **kwargs):
-            result_queue = queue.Queue()
-            self.call_queue.put((result_queue, self.func, args, kwargs))
-            (exc_info, result) = result_queue.get()
-            if exc_info is None:
-                return result
-            else:
-                raise exc_info[1].with_traceback(exc_info[2])
-
-    class SerializerObjectProxy():
-        def __init__(self, obj_or_type, *args, **kwargs):
-            self.__object = obj_or_type
-            if isinstance(obj_or_type, type):
-                classname = obj_or_type.__name__
-            else:
-                classname = obj_or_type.__class__.__name__
-            self.__call_queue = queue.Queue()
-            self.__thread = SerializerThread(classname, self.__call_queue)
-            self.__thread.daemon = True
-            self.__thread.start()
-            self._thread_safe = True
-
-        def __getattr__(self, key):
-            # If the attribute is a function, we want to return a
-            # proxy that will perform the call through the serializer
-            # when called.  Otherwise, we want to return the value
-            # directly.  This means we need to grab the attribute once,
-            # and therefore self.__object.__getattr__ may be called
-            # in an unsafe way, from the caller's thread.
-            attr = getattr(self.__object, key)
-            if not callable(attr):
-                # It's not callable, so perform the getattr from within
-                # the serializer thread, then return its value.
-                # That may differ from the "attr" value we just grabbed
-                # from here, due to forced ordering in the serializer.
-                getter = SerializerCallProxy(self.__call_queue, getattr, self)
-                return getter(self.__object, key)
-            else:
-                # It is callable, so return an object that will proxy through
-                # the serializer when called.
-                r = SerializerCallProxy(self.__call_queue, attr, self)
-                return r
-
-        # For an interable object, on __iter__(), save the object's
-        # iterator and return this proxy.  On next(), call the object's
-        # iterator through this proxy.
-        def __iter__(self):
-            attr = getattr(self.__object, "__iter__")
-            self.__iter = SerializerCallProxy(self.__call_queue, attr, self)()
-            return self
-
-        def __next__(self):
-            return SerializerCallProxy(self.__call_queue,
-                                       self.__iter.__next__, self)()
-
-        def __getitem__(self, key):
-            return self.__getattr__("__getitem__")(key)
-
-        def __call__(self, *args, **kwargs):
-            """Call this to instantiate the type, if a type was passed
-            to serializer_proxy.  Otherwise, pass the call through."""
-            ret = SerializerCallProxy(self.__call_queue,
-                                      self.__object, self)(*args, **kwargs)
-            if isinstance(self.__object, type):
-                # Instantiation
-                self.__object = ret
-                return self
-            return ret
-
-        def __del__(self):
-            try:
-                # Signal thread to exit, but don't wait for it.
-                self.__call_queue.put((None, None, None, None))
-            except:
-                pass
-
-    return SerializerObjectProxy(obj_or_type)

nilmdb/utils/sort.py

@@ -1,19 +0,0 @@
-import re
-
-
-def sort_human(items, key=None):
-    """Human-friendly sort (/stream/2 before /stream/10)"""
-    def to_num(val):
-        try:
-            return int(val)
-        except Exception:
-            return val
-
-    def human_key(text):
-        if key:
-            text = key(text)
-        # Break into character and numeric chunks.
-        chunks = re.split(r'([0-9]+)', text)
-        return [to_num(c) for c in chunks]
-
-    return sorted(items, key=human_key)

nilmdb/utils/threadsafety.py

@@ -1,97 +0,0 @@
-import threading
-from nilmdb.utils.printf import sprintf
-
-
-def verify_proxy(obj_or_type, check_thread=True,
-                 check_concurrent=True):
-    """Wrap the given object or type in a VerifyObjectProxy.
-
-    Returns a VerifyObjectProxy that proxies all method calls to the
-    given object, as well as attribute retrievals.
-
-    When calling methods, the following checks are performed.  On
-    failure, an exception is raised.
-
-    check_thread = True     # Fail if two different threads call methods.
-    check_concurrent = True # Fail if two functions are concurrently
-                            # run through this proxy
-    """
-    class Namespace():
-        pass
-
-    class VerifyCallProxy():
-        def __init__(self, func, parent_namespace):
-            self.func = func
-            self.parent_namespace = parent_namespace
-
-        def __call__(self, *args, **kwargs):
-            p = self.parent_namespace
-            this = threading.current_thread()
-            try:
-                callee = self.func.__name__
-            except AttributeError:
-                callee = "???"
-
-            if p.thread is None:
-                p.thread = this
-                p.thread_callee = callee
-
-            if check_thread and p.thread != this:
-                err = sprintf("unsafe threading: %s called %s.%s,"
-                              " but %s called %s.%s",
-                              p.thread.name, p.classname, p.thread_callee,
-                              this.name, p.classname, callee)
-                raise AssertionError(err)
-
-            need_concur_unlock = False
-            if check_concurrent:
-                if not p.concur_lock.acquire(False):
-                    err = sprintf("unsafe concurrency: %s called %s.%s "
-                                  "while %s is still in %s.%s",
-                                  this.name, p.classname, callee,
-                                  p.concur_tname, p.classname, p.concur_callee)
-                    raise AssertionError(err)
-                else:
-                    p.concur_tname = this.name
-                    p.concur_callee = callee
-                    need_concur_unlock = True
-
-            try:
-                ret = self.func(*args, **kwargs)
-            finally:
-                if need_concur_unlock:
-                    p.concur_lock.release()
-            return ret
-
-    class VerifyObjectProxy():
-        def __init__(self, obj_or_type, *args, **kwargs):
-            p = Namespace()
-            self.__ns = p
-            p.thread = None
-            p.thread_callee = None
-            p.concur_lock = threading.Lock()
-            p.concur_tname = None
-            p.concur_callee = None
-            self.__obj = obj_or_type
-            if isinstance(obj_or_type, type):
-                p.classname = self.__obj.__name__
-            else:
-                p.classname = self.__obj.__class__.__name__
-
-        def __getattr__(self, key):
-            attr = getattr(self.__obj, key)
-            if not callable(attr):
-                return VerifyCallProxy(getattr, self.__ns)(self.__obj, key)
-            return VerifyCallProxy(attr, self.__ns)
-
-        def __call__(self, *args, **kwargs):
-            """Call this to instantiate the type, if a type was passed
-            to verify_proxy.  Otherwise, pass the call through."""
-            ret = VerifyCallProxy(self.__obj, self.__ns)(*args, **kwargs)
-            if isinstance(self.__obj, type):
-                # Instantiation
-                self.__obj = ret
-                return self
-            return ret
-
-    return VerifyObjectProxy(obj_or_type)

nilmdb/utils/time.py

@@ -1,148 +0,0 @@
-import re
-import time
-import datetime_tz
-
-# Range
-min_timestamp = (-2**63)
-max_timestamp = (2**63 - 1)
-
-# Smallest representable step
-epsilon = 1
-
-
-def string_to_timestamp(string):
-    """Convert a string that represents an integer number of microseconds
-    since epoch."""
-    try:
-        # Parse a string like "1234567890123456" and return an integer
-        return int(string)
-    except ValueError:
-        # Try parsing as a float, in case it's "1234567890123456.0"
-        return int(round(float(string)))
-
-
-def timestamp_to_string(timestamp):
-    """Convert a timestamp (integer microseconds since epoch) to a string"""
-    if isinstance(timestamp, float):
-        return str(int(round(timestamp)))
-    else:
-        return str(timestamp)
-
-
-def timestamp_to_bytes(timestamp):
-    """Convert a timestamp (integer microseconds since epoch) to a Python
-    bytes object"""
-    return timestamp_to_string(timestamp).encode('utf-8')
-
-
-def timestamp_to_human(timestamp):
-    """Convert a timestamp (integer microseconds since epoch) to a
-    human-readable string, using the local timezone for display
-    (e.g. from the TZ env var)."""
-    if timestamp == min_timestamp:
-        return "(minimum)"
-    if timestamp == max_timestamp:
-        return "(maximum)"
-    dt = datetime_tz.datetime_tz.fromtimestamp(timestamp_to_unix(timestamp))
-    return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
-
-
-def unix_to_timestamp(unix):
-    """Convert a Unix timestamp (floating point seconds since epoch)
-    into a NILM timestamp (integer microseconds since epoch)"""
-    return int(round(unix * 1e6))
-
-
-def timestamp_to_unix(timestamp):
-    """Convert a NILM timestamp (integer microseconds since epoch)
-    into a Unix timestamp (floating point seconds since epoch)"""
-    return timestamp / 1e6
-
-
-seconds_to_timestamp = unix_to_timestamp
-timestamp_to_seconds = timestamp_to_unix
-
-
-def rate_to_period(hz, cycles=1):
-    """Convert a rate (in Hz) to a period (in timestamp units).
-    Returns an integer."""
-    period = unix_to_timestamp(cycles) / float(hz)
-    return int(round(period))
-
-
-def parse_time(toparse):
-    """
-    Parse a free-form time string and return a nilmdb timestamp
-    (integer microseconds since epoch).  If the string doesn't contain a
-    timestamp, the current local timezone is assumed (e.g. from the TZ
-    env var).
-    """
-    if toparse == "min":
-        return min_timestamp
-    if toparse == "max":
-        return max_timestamp
-
-    # If it starts with @, treat it as a NILM timestamp
-    # (integer microseconds since epoch)
-    try:
-        if toparse[0] == '@':
-            return int(toparse[1:])
-    except (ValueError, KeyError, IndexError):
-        pass
-
-    # If string isn't "now" and doesn't contain at least 4 digits,
-    # consider it invalid.  smartparse might otherwise accept
-    # empty strings and strings with just separators.
-    if toparse != "now" and len(re.findall(r"\d", toparse)) < 4:
-        raise ValueError("not enough digits for a timestamp")
-
-    # Try to just parse the time as given
-    try:
-        return unix_to_timestamp(datetime_tz.datetime_tz.
-                                 smartparse(toparse).totimestamp())
-    except (ValueError, OverflowError, TypeError):
-        pass
-
-    # If it's parseable as a float, treat it as a Unix or NILM
-    # timestamp based on its range.
-    try:
-        val = float(toparse)
-        # range is from about year 2001 - 2128
-        if 1e9 < val < 5e9:
-            return unix_to_timestamp(val)
-        if 1e15 < val < 5e15:
-            return val
-    except ValueError:
-        pass
-
-    # Try to extract a substring in a condensed format that we expect
-    # to see in a filename or header comment
-    res = re.search(r"(^|[^\d])("            # non-numeric or SOL
-                    r"(199\d|2\d\d\d)"       # year
-                    r"[-/]?"                 # separator
-                    r"(0[1-9]|1[012])"       # month
-                    r"[-/]?"                 # separator
-                    r"([012]\d|3[01])"       # day
-                    r"[-T ]?"                # separator
-                    r"([01]\d|2[0-3])"       # hour
-                    r"[:]?"                  # separator
-                    r"([0-5]\d)"             # minute
-                    r"[:]?"                  # separator
-                    r"([0-5]\d)?"            # second
-                    r"([-+]\d\d\d\d)?"       # timezone
-                    r")", toparse)
-    if res is not None:
-        try:
-            return unix_to_timestamp(datetime_tz.datetime_tz.
-                                     smartparse(res.group(2)).totimestamp())
-        except ValueError:
-            pass
-
-    # Could also try to successively parse substrings, but let's
-    # just give up for now.
-    raise ValueError("unable to parse timestamp")
-
-
-def now():
-    """Return current timestamp"""
-    return unix_to_timestamp(time.time())

nilmdb/utils/timer.py

@@ -1,22 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Simple timer to time a block of code, for optimization debugging
-# use like:
-#   with nilmdb.utils.Timer("flush"):
-#       foo.flush()
-
-import contextlib
-import time
-
-
-@contextlib.contextmanager
-def Timer(name=None, tosyslog=False):
-    start = time.time()
-    yield
-    elapsed = int((time.time() - start) * 1000)
-    msg = (name or 'elapsed') + ": " + str(elapsed) + " ms"
-    if tosyslog:
-        import syslog
-        syslog.syslog(msg)
-    else:
-        print(msg)

nilmdb/utils/timestamper.py

@@ -1,103 +0,0 @@
-"""File-like objects that add timestamps to the input lines"""
-
-from nilmdb.utils.printf import sprintf
-import nilmdb.utils.time
-
-
-class Timestamper():
-    """A file-like object that adds timestamps to lines of an input file."""
-    def __init__(self, infile, ts_iter):
-        """file: filename, or another file-like object
-           ts_iter: iterator that returns a timestamp string for
-           each line of the file"""
-        if isinstance(infile, str):
-            self.file = open(infile, "rb")
-        else:
-            self.file = infile
-        self.ts_iter = ts_iter
-
-    def close(self):
-        self.file.close()
-
-    def readline(self, *args):
-        while True:
-            line = self.file.readline(*args)
-            if not line:
-                return b""
-            if line[0:1] == b'#':
-                continue
-            # For some reason, coverage on python 3.8 reports that
-            # we never hit this break, even though we definitely do.
-            break  # pragma: no cover
-        try:
-            return next(self.ts_iter) + line
-        except StopIteration:
-            return b""
-
-    def readlines(self, size=None):
-        out = b""
-        while True:
-            line = self.readline()
-            out += line
-            if not line or (size and len(out) >= size):
-                break
-        return out
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        result = self.readline()
-        if not result:
-            raise StopIteration
-        return result
-
-
-class TimestamperRate(Timestamper):
-    """Timestamper that uses a start time and a fixed rate"""
-    def __init__(self, infile, start, rate, end=None):
-        """
-        file: file name or object
-
-        start: Unix timestamp for the first value
-
-        rate: 1/rate is added to the timestamp for each line
-
-        end: If specified, raise StopIteration before outputting a value
-             greater than this."""
-        timestamp_to_bytes = nilmdb.utils.time.timestamp_to_bytes
-        rate_to_period = nilmdb.utils.time.rate_to_period
-
-        def iterator(start, rate, end):
-            n = 0
-            rate = float(rate)
-            while True:
-                now = start + rate_to_period(rate, n)
-                if end and now >= end:
-                    return
-                yield timestamp_to_bytes(now) + b" "
-                n += 1
-        Timestamper.__init__(self, infile, iterator(start, rate, end))
-        self.start = start
-        self.rate = rate
-
-    def __str__(self):
-        return sprintf("TimestamperRate(..., start=\"%s\", rate=%g)",
-                       nilmdb.utils.time.timestamp_to_human(self.start),
-                       self.rate)
-
-
-class TimestamperNow(Timestamper):
-    """Timestamper that uses current time"""
-    def __init__(self, infile):
-        timestamp_to_bytes = nilmdb.utils.time.timestamp_to_bytes
-        get_now = nilmdb.utils.time.now
-
-        def iterator():
-            while True:
-                yield timestamp_to_bytes(get_now()) + b" "
-
-        Timestamper.__init__(self, infile, iterator())
-
-    def __str__(self):
-        return "TimestamperNow(...)"

pytables-test/Makefile

@@ -0,0 +1,5 @@
+all:
+	time python test-indexed-read.py
+
+clean:
+	rm -f *pyc

pytables-test/README.jim

@@ -0,0 +1,2 @@
+New version from:
+  http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=641485#15

pytables-test/TODO.txt

@@ -0,0 +1,12 @@
+- Make CherryPy server that can handle simple GET/POST,
+  and a httplib client that can talk to that server.
+  Steps:
+  - Make server handle GET
+  - Make client send request, get response
+  - Add request streaming to server
+  - Add request streaming to client
+  - Make server handle POST
+  - Make client send request, get response
+  - Add request streaming to server
+  - Add request streaming to client
+  - Integrate into a server process that also keeps database open.

pytables-test/indexing-notes

@@ -0,0 +1,3 @@
+Indexing time64 doesn't seem to work -- needed to do "time >= 1243052015" even though the actual database times
+should be something like 1243052015.847000.  Let's switch to just using a 64-bit integer counting e.g.
+microseconds since 1970-01-01 

pytables-test/sample-query

@@ -0,0 +1,3 @@
+timestamp > 1243052015
+took 394.5 minutes in vitables
+(2340 rows matched)

pytables-test/server.py

@@ -0,0 +1,53 @@
+import sys
+import tables
+import nilmdb
+
+try:
+    import cherrypy
+    cherrypy.tools.json_out
+except:
+    sys.stderr.write("Cherrypy 3.2+ required\n")
+    sys.exit(1)
+
+class NilmApp:
+    def __init__(self, db):
+        self.db = db
+
+class Root(NilmApp):
+    """NILM Database"""
+
+    server_version = "1.0"
+
+    @cherrypy.expose
+    def index(self):
+        raise cherrypy.NotFound()
+
+    @cherrypy.expose
+    def favicon_ico(self):
+        raise cherrypy.NotFound()
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def version(self):
+        return self.server_version
+
+class Stream(NilmApp):
+    """Stream operations"""
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def list(self):
+        return 
+    
+cherrypy.config.update({
+    'server.socket_host': '127.0.0.1',
+    'server.socket_port': 12380
+    })
+
+db = nilmdb.nilmdb()
+cherrypy.tree.mount(Root(db), "/")
+cherrypy.tree.mount(Stream(db), "/stream")
+
+if __name__ == "__main__":
+    cherrypy.engine.start()
+    cherrypy.engine.block()

pytables-test/speed-pytables.py

@@ -0,0 +1,16 @@
+import tables
+import numpy
+
+class RawSample(tables.IsDescription):
+    timestamp = tables.UInt64Col()
+    voltage   = tables.UInt16Col(shape = 3)
+    current   = tables.UInt16Col(shape = 3)
+
+h5file = tables.openFile("test.h5", mode = "w", title = "Test")
+group = h5file.createGroup("/", "raw", "Raw Data")
+table = h5file.createTable(group, "nilm1", RawSample, "NILM 1")
+
+print repr(h5file)
+
+# write rows
+

pytables-test/test-indexing.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+from tables import *
+import re
+import time
+
+# A class to describe our data
+class PrepData(IsDescription):
+    timestamp = Int64Col()
+    p1 = Float32Col()
+    q1 = Float32Col()
+    p3 = Float32Col()
+    q3 = Float32Col()
+    p5 = Float32Col()
+    q5 = Float32Col()
+    p7 = Float32Col()
+    q7 = Float32Col()
+
+filename = "test.h5"
+h5file = openFile(filename, mode = "w", title = "NILM Test")
+
+group = h5file.createGroup("/", "newton", "Newton school")
+table = h5file.createTable(group, "prep", PrepData, "Prep Data", expectedrows = 120 * 86400 * 90)
+
+table.cols.timestamp.createIndex()
+
+for i in range(0, 80):
+    # Open file
+    data = open("data/alldata")
+    count = 0
+    oldtime = time.time()
+    prep = table.row
+    for line in data:
+        count = count + 1
+        if count % 1000000 == 0:
+            print str(i) + ": " + str((time.time() - oldtime)) + ", total " + str(count/1000000) + "m lines"
+            oldtime = time.time()
+        v = re.split('\s+', line)
+        prep['timestamp'] = int(v[0]) + 500000000 * i
+        prep['p1'] = v[1]
+        prep['q1'] = v[2]
+        prep['p3'] = v[3]
+        prep['q3'] = v[4]
+        prep['p5'] = v[5]
+        prep['q5'] = v[6]
+        prep['p7'] = v[7]
+        prep['q7'] = v[8]
+        prep.append()
+    data.close()
+
+h5file.close()
+
+
+

pytables-test/test-write.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+from tables import *
+import re
+import time
+
+# A class to describe our data
+class PrepData(IsDescription):
+    timestamp = Time64Col()
+    p1 = Float32Col()
+    q1 = Float32Col()
+    p3 = Float32Col()
+    q3 = Float32Col()
+    p5 = Float32Col()
+    q5 = Float32Col()
+    p7 = Float32Col()
+    q7 = Float32Col()
+
+filename = "test.h5"
+h5file = openFile(filename, mode = "w", title = "NILM Test")
+
+group = h5file.createGroup("/", "newton", "Newton school")
+table = h5file.createTable(group, "prep", PrepData, "Prep Data")
+
+table.cols.timestamp.createIndex()
+
+for i in range(0, 80):
+    # Open file
+    data = open("data/alldata")
+    count = 0
+    oldtime = time.time()
+    prep = table.row
+    for line in data:
+        count = count + 1
+        if count % 1000000 == 0:
+            print str(i) + ": " + str((time.time() - oldtime)) + ", total " + str(count/1000000) + "m lines"
+            oldtime = time.time()
+        v = re.split('\s+', line)
+        prep['timestamp'] = float(v[0]) / 1000.0 + 500000 * i
+        prep['p1'] = v[1]
+        prep['q1'] = v[2]
+        prep['p3'] = v[3]
+        prep['q3'] = v[4]
+        prep['p5'] = v[5]
+        prep['q5'] = v[6]
+        prep['p7'] = v[7]
+        prep['q7'] = v[8]
+        prep.append()
+    data.close()
+
+h5file.close()
+
+
+

requirements.txt

@@ -1,41 +0,0 @@
-argcomplete==1.12.0
-CherryPy==18.6.0
-coverage==5.2.1
-Cython==0.29.21
-decorator==4.4.2
-fallocate==1.6.4
-flake8==3.8.3
-nose==1.3.7
-numpy==1.19.1
-progressbar==2.5
-psutil==5.7.2
-python-datetime-tz==0.5.4
-python-dateutil==2.8.1
-requests==2.24.0
-tz==0.2.2
-yappi==1.2.5
-
-## The following requirements were added by pip freeze:
-beautifulsoup4==4.9.1
-certifi==2020.6.20
-chardet==3.0.4
-cheroot==8.4.2
-idna==2.10
-jaraco.classes==3.1.0
-jaraco.collections==3.0.0
-jaraco.functools==3.0.1
-jaraco.text==3.2.0
-mccabe==0.6.1
-more-itertools==8.4.0
-portend==2.6
-pycodestyle==2.6.0
-pyflakes==2.2.0
-pytz==2020.1
-six==1.15.0
-soupsieve==2.0.1
-tempora==4.0.0
-urllib3==1.25.10
-waitress==1.4.4
-WebOb==1.8.6
-WebTest==2.0.35
-zc.lockfile==2.0

setup.cfg

@@ -1,59 +1,9 @@
-[aliases]
-test = nosetests
-
 [nosetests]
-# Note: values must be set to 1, and have no comments on the same line,
-# for "python setup.py nosetests" to work correctly.
 nocapture=1
-# Comment this out to see CherryPy logs on failure:
-nologcapture=1
 with-coverage=1
 cover-inclusive=1
 cover-package=nilmdb
 cover-erase=1
-# this works, puts html output in cover/ dir:
-# cover-html=1
-#debug=nose
-#debug-log=nose.log
 stop=1
 verbosity=2
-tests=tests
-#tests=tests/test_threadsafety.py
-#tests=tests/test_bulkdata.py
-#tests=tests/test_mustclose.py
-#tests=tests/test_lrucache.py
-#tests=tests/test_cmdline.py
-#tests=tests/test_layout.py
-#tests=tests/test_rbtree.py
-#tests=tests/test_interval.py
-#tests=tests/test_rbtree.py,tests/test_interval.py
-#tests=tests/test_interval.py
-#tests=tests/test_client.py
-#tests=tests/test_timestamper.py
-#tests=tests/test_serializer.py
-#tests=tests/test_iteratorizer.py
-#tests=tests/test_client.py:TestClient.test_client_nilmdb
-#tests=tests/test_nilmdb.py
-#with-profile=1
-#profile-sort=time
-##profile-restrict=10  # doesn't work right, treated as string or something
 
-[versioneer]
-VCS=git
-style=pep440
-versionfile_source=nilmdb/_version.py
-versionfile_build=nilmdb/_version.py
-tag_prefix=nilmdb-
-parentdir_prefix=nilmdb-
-
-[flake8]
-exclude=_version.py
-extend-ignore=E731
-per-file-ignores=__init__.py:F401,E402 \
-        serializer.py:E722 \
-        mustclose.py:E722 \
-        fsck.py:E266
-
-[pylint]
-ignore=_version.py
-disable=C0103,C0111,R0913,R0914

setup.py

@@ -1,64 +1,9 @@
-#!/usr/bin/env python3
+#!/usr/bin/python
 
-# To release a new version, tag it:
+from distutils.core import setup
-#   git tag -a nilmdb-1.1 -m "Version 1.1"
-#   git push --tags
-# Then just package it up:
-#   python3 setup.py sdist
 
-import sys
-import os
-from setuptools import setup
-from distutils.extension import Extension
-
-# Versioneer manages version numbers from git tags.
-# https://github.com/warner/python-versioneer
-import versioneer
-
-# External modules that need to be built
-ext_modules = [ Extension('nilmdb.server.rocket', ['nilmdb/server/rocket.c' ]) ]
-
-# Use Cython.
-cython_modules = [ 'nilmdb.server.interval', 'nilmdb.server.rbtree' ]
-import Cython
-from Cython.Build import cythonize
-for modulename in cython_modules:
-    filename = modulename.replace('.','/')
-    ext_modules.extend(cythonize(filename + ".pyx"))
-
-# Get list of requirements to use in `install_requires` below.  Note
-# that we don't make a distinction between things that are actually
-# required for end-users vs developers (or use `test_requires` or
-# anything else) -- just install everything for simplicity.
-install_requires = open('requirements.txt').readlines()
-
-# Run setup
 setup(name = 'nilmdb',
-      version = versioneer.get_version(),
+      version = '1.0',
-      cmdclass = versioneer.get_cmdclass(),
+      scripts = [ 'bin/nilm-test.py' ],
-      url = 'https://git.jim.sh/nilm/nilmdb.git',
+      packages = [ 'nilmdb' ],
-      author = 'Jim Paris',
-      description = "NILM Database",
-      long_description = "NILM Database",
-      license = "Proprietary",
-      author_email = 'jim@jtan.com',
-      setup_requires = [ 'setuptools' ],
-      install_requires = install_requires,
-      packages = [ 'nilmdb',
-                   'nilmdb.utils',
-                   'nilmdb.server',
-                   'nilmdb.client',
-                   'nilmdb.cmdline',
-                   'nilmdb.scripts',
-                   'nilmdb.fsck',
-                   ],
-      entry_points = {
-          'console_scripts': [
-              'nilmtool = nilmdb.scripts.nilmtool:main',
-              'nilmdb-server = nilmdb.scripts.nilmdb_server:main',
-              'nilmdb-fsck = nilmdb.scripts.nilmdb_fsck:main',
-              ],
-          },
-      ext_modules = ext_modules,
-      zip_safe = False,
       )

test/Makefile

@@ -0,0 +1,5 @@
+all:
+	python speed-readascii.py
+
+clean:
+	rm -f *pyc

test/printf.py

@@ -0,0 +1,4 @@
+from __future__ import print_function
+def printf(str, *args):
+    print(str % args, end='')
+    

test/speed-readascii.py

@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+from printf import printf
+import time
+import re
+import numpy as np
+import itertools
+
+class Timer():
+    def __init__(self, arg):
+        self.arg = arg
+    def __enter__(self): self.start = time.time()
+    def __exit__(self, *args): printf("%s: %f lines/sec\n", self.arg, 1e6 / (time.time() - self.start))
+
+def test_split():
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [0]*6
+        tmp = [ int(i) for i in line.partition('#')[0].split() ]
+        out[0:len(tmp)] = tmp
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_split2():
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [0]*6
+        tmp = [ int(i,10) for i in line.partition('#')[0].split() ]
+        out[0:len(tmp)] = tmp
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_regex():
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [0]*6
+        tmp = [ int(x) for x in re.findall('(\d+)\s+',line.partition('#')[0]) ]
+        out[0:len(tmp)] = tmp
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_bigregex():
+    regex = re.compile('^(?:\s*)' + '(?:(\d+)\s+)?' * 6)
+    for n, line in enumerate(open('1m.raw', 'r')):
+        out = [ int(x or 0) for x in re.match(regex, line).groups() ]
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_numpy():
+    out = np.genfromtxt(open('1m.raw', 'r'),
+                        dtype = np.dtype('i2,i2,i2,i2,i2,i2'))
+    
+with Timer("numpy"):
+    test_numpy() # 106k/sec
+
+with Timer("regex"):
+    test_regex() # 121k/sec
+
+with Timer("split"):
+    test_split() # 219k/sec
+
+with Timer("split2"):
+    test_split2() # 328k/sec
+
+with Timer("bigregex"):
+    test_bigregex() # 130k/sec
+
+# The "int" operation takes quite a while -- int(x,10) is twice as fast
+# Perl does about 500k/sec
+

test/speed-readbinary.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+
+from printf import printf
+import time
+import re
+import numpy as np
+import itertools
+import struct
+import array
+import os
+import mmap
+
+class Timer():
+    def __init__(self, arg):
+        self.arg = arg
+    def __enter__(self): self.start = time.time()
+    def __exit__(self, *args): printf("%s: %f klines/sec\n", self.arg, 1e3 / (time.time() - self.start))
+
+def test_struct1():
+    """read with struct.unpack"""
+    f = open('1m.bin', 'rb')
+    f.seek(0,os.SEEK_END)
+    filesize = f.tell()
+    f.seek(0,os.SEEK_SET)
+    packer = struct.Struct('!dHHHHHH')
+    items = filesize / packer.size
+    for n in xrange(items):
+        s = f.read(packer.size)
+        out = packer.unpack(s)
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_struct2():
+    """read with struct.unpack, convert to string"""
+    f = open('1m.bin', 'rb')
+    f.seek(0,os.SEEK_END)
+    filesize = f.tell()
+    f.seek(0,os.SEEK_SET)
+    packer = struct.Struct('!dHHHHHH')
+    items = filesize / packer.size
+    for n in xrange(items):
+        s = f.read(packer.size)
+        out = packer.unpack(s)
+        x = str(out)
+        if (n % 100000 == 0):
+            printf("line %d = %s\n", n, str(out))
+
+def test_mmap():
+    """struct.unpack with mmap"""
+    with open('1m.bin', 'rb') as f:
+        f.seek(0,os.SEEK_END)
+        filesize = f.tell()
+        f.seek(0,os.SEEK_SET)
+        m = mmap.mmap(f.fileno(), filesize, access=mmap.ACCESS_READ)
+        packer = struct.Struct('!dHHHHHH')
+        items = filesize / packer.size
+        for n in xrange(items):
+            out = packer.unpack(m[packer.size*n : packer.size*(n+1)])
+            if (n % 100000 == 0):
+                printf("line %d = %s\n", n, str(out))
+
+with Timer("mmap"):
+    test_mmap()  # 1600k
+
+with Timer("struct1"):
+    test_struct1()  # 1460k
+
+with Timer("struct2"):
+    test_struct2()  # 210k
+
+# Reading from the file is again much quicker than converting to string
+# Use mmap, it's good
+
+

test/speed-writebinary.py

@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+from printf import printf
+import time
+import re
+import numpy as np
+import itertools
+import struct
+import array
+
+class Timer():
+    def __init__(self, arg):
+        self.arg = arg
+    def __enter__(self): self.start = time.time()
+    def __exit__(self, *args): printf("%s: %f klines/sec\n", self.arg, 1e3 / (time.time() - self.start))
+
+def read_ascii():
+    for n in xrange(1000000):
+        yield (1234, 2345, 3456, 4576, 5678, 6789)
+#    for n, line in enumerate(open('1m.raw', 'r')):
+#        out = [0]*6
+#        tmp = [ int(i,10) for i in line.partition('#')[0].split() ]
+#        out[0:len(tmp)] = tmp
+#        if (n % 100000 == 0):
+#            printf("line %d = %s\n", n, str(out))
+#        yield out
+
+def test_struct1():
+    """write with struct.pack"""
+    f = open('1m.bin', 'wb')
+    for out in read_ascii():
+        s = struct.pack('!HHHHHH', *out)
+        f.write(s)
+
+def test_struct2():
+    """use constant format string"""
+    f = open('1m.bin', 'wb')
+    packer = struct.Struct('!HHHHHH')
+    for out in read_ascii():
+        f.write(packer.pack(*out))
+    f.close()
+    printf("size was %d\n", packer.size)
+
+def test_struct3():
+    """like struct1, with timestamp"""
+    f = open('1m.bin', 'wb')
+    for out in read_ascii():
+        s = struct.pack('!dHHHHHH', time.time(), *out)
+        f.write(s)
+
+def test_struct4():
+    """like struct2, with timestamp"""
+    f = open('1m.bin', 'wb')
+    packer = struct.Struct('!dHHHHHH')
+    for out in read_ascii():
+        f.write(packer.pack(time.time(), *out))
+    f.close()
+    printf("size was %d\n", packer.size)
+
+#raise Exception('done')
+
+with Timer("struct1"):
+    test_struct1() # 1089k
+
+with Timer("struct2"):
+    test_struct2() # 1249k
+
+with Timer("struct3"):
+    test_struct3() # 845k
+
+with Timer("struct4"):
+    test_struct4() # 922k
+
+# This seems fast enough for writing new data, since it's faster than
+# we read ascii data anyway.  Use e.g. struct4
+

test/test-struct-pack.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+import struct
+import mmap
+
+f = open("test.dat", "rb+")
+mm = mmap.mmap(f.fileno(),3)
+
+print len(mm)
+print "first 3 bytes: " + mm[0:3];
+

test/test.dat

@@ -0,0 +1 @@
+

















test/todo.md

@@ -0,0 +1,7 @@
+- Have a class representing the file contents
+- Looks like an array
+  - len(), get(), index
+  - some form of bisect search
+  - get_extents = return [0].timestamp, [-1].timestamp
+  - 
+- Can append?  Sure, why not.  Just write to the file, extend mmap accordingly.

tests/data/extract-1

@@ -1,124 +0,0 @@
-# path: /newton/prep
-# layout: float32_8
-# start: Fri, 23 Mar 2012 10:00:30.000000 +0000
-# end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-1332496830000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-2

@@ -1,119 +0,0 @@
-1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-3

@@ -1 +0,0 @@
-1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03

tests/data/extract-4

@@ -1,2 +0,0 @@
-1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03

tests/data/extract-5

@@ -1,124 +0,0 @@
-# path: /newton/prep
-# layout: float32_8
-# start: Fri, 23 Mar 2012 10:00:30.000000 +0000
-# end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-6

@@ -1,120 +0,0 @@
-2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-7

@@ -1,124 +0,0 @@
-# path: /newton/prep
-# layout: float32_8
-# start: 1332496830000000
-# end: 1332496830999000
-1332496830000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-1332496830008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-8

@@ -1,28 +0,0 @@
-# interval-start 1332496919900000
-1332496919900000 2.523050e+05 2.254020e+05 4.779410e+03 3.638030e+03 8.138070e+03 4.334460e+03 1.083780e+03 3.743730e+03
-1332496919908333 2.551190e+05 2.237870e+05 5.965640e+03 2.076350e+03 9.468790e+03 3.693880e+03 1.247860e+03 3.393680e+03
-1332496919916667 2.616370e+05 2.247980e+05 4.848970e+03 2.315620e+03 9.323300e+03 4.225460e+03 1.805780e+03 2.593050e+03
-1332496919925000 2.606460e+05 2.251300e+05 3.061360e+03 3.951840e+03 7.662910e+03 5.341410e+03 1.986520e+03 2.276780e+03
-1332496919933333 2.559710e+05 2.235030e+05 4.096030e+03 3.296970e+03 7.827080e+03 5.452120e+03 2.492520e+03 2.929450e+03
-1332496919941667 2.579260e+05 2.217080e+05 5.472320e+03 1.555700e+03 8.495760e+03 4.491140e+03 2.379780e+03 3.741710e+03
-1332496919950000 2.610180e+05 2.242350e+05 4.669770e+03 1.876190e+03 8.366680e+03 3.677510e+03 9.021690e+02 3.549040e+03
-1332496919958333 2.569150e+05 2.274650e+05 2.785070e+03 3.751930e+03 7.440320e+03 3.964860e+03 -3.227860e+02 2.460890e+03
-1332496919966667 2.509510e+05 2.262000e+05 3.772710e+03 3.131950e+03 8.159860e+03 4.539860e+03 7.375190e+02 2.126750e+03
-1332496919975000 2.556710e+05 2.223720e+05 5.826200e+03 8.715560e+02 9.120240e+03 4.545110e+03 2.804310e+03 2.721000e+03
-1332496919983333 2.649730e+05 2.214860e+05 5.839130e+03 4.659180e+02 8.628300e+03 3.934870e+03 2.972490e+03 3.773730e+03
-1332496919991667 2.652170e+05 2.233920e+05 3.718770e+03 2.834970e+03 7.209900e+03 3.460260e+03 1.324930e+03 4.075960e+03
-# interval-end 1332496919991668
-# interval-start 1332496920000000
-1332496920000000 2.564370e+05 2.244300e+05 4.011610e+03 3.475340e+03 7.495890e+03 3.388940e+03 2.613970e+02 3.731260e+03
-1332496920008333 2.539630e+05 2.241670e+05 5.621070e+03 1.548010e+03 9.165170e+03 3.522930e+03 1.058930e+03 2.996960e+03
-1332496920016667 2.585080e+05 2.249300e+05 6.011400e+03 8.188660e+02 9.039950e+03 4.482440e+03 2.490390e+03 2.679340e+03
-1332496920025000 2.596270e+05 2.260220e+05 4.474500e+03 2.423020e+03 7.414190e+03 5.071970e+03 2.439380e+03 2.962960e+03
-1332496920033333 2.551870e+05 2.246320e+05 4.738570e+03 3.398040e+03 7.395120e+03 4.726450e+03 1.839030e+03 3.393530e+03
-1332496920041667 2.571020e+05 2.216230e+05 6.144130e+03 1.441090e+03 8.756480e+03 3.495320e+03 1.869940e+03 3.752530e+03
-1332496920050000 2.636530e+05 2.217700e+05 6.221770e+03 7.389620e+02 9.547600e+03 2.666820e+03 1.462660e+03 3.332570e+03
-1332496920058333 2.636130e+05 2.252560e+05 4.477120e+03 2.437450e+03 8.510210e+03 3.855630e+03 9.594420e+02 2.387180e+03
-1332496920066667 2.553500e+05 2.262640e+05 4.283720e+03 3.923940e+03 7.912470e+03 5.466520e+03 1.284990e+03 2.093720e+03
-1332496920075000 2.527270e+05 2.246090e+05 5.851930e+03 2.491980e+03 8.540630e+03 5.623050e+03 2.339780e+03 3.007140e+03
-1332496920083333 2.584750e+05 2.235780e+05 5.924870e+03 1.394480e+03 8.779620e+03 4.544180e+03 2.132030e+03 3.849760e+03
-1332496920091667 2.615630e+05 2.246090e+05 4.336140e+03 2.455750e+03 8.055380e+03 3.469110e+03 6.278730e+02 3.664200e+03
-# interval-end 1332496920100000