Tune sqlite to use write-ahead-logging

Enable the following pragmas: synchronous=NORMAL, journal_mode=WAL.
This offers a significant speedup to INSERT times compared to
synchronous=FULL, and is roughly the same as synchronous=OFF
but should be a bit safer.

.gitignore

@@ -8,11 +8,9 @@ docs/*.html
 build/
 *.pyc
 nilmdb/server/interval.c
-nilmdb/server/interval.so
 nilmdb/server/layout.c
-nilmdb/server/layout.so
 nilmdb/server/rbtree.c
-nilmdb/server/rbtree.so
+*.so
 
 # Setup junk
 dist/

Makefile

@@ -14,6 +14,9 @@ sdist:
 install:
 	python setup.py install
 
+develop:
+	python setup.py develop
+
 docs:
 	make -C docs
 
@@ -23,6 +26,7 @@ lint:
 test:
 ifeq ($(INSIDE_EMACS), t)
 # Use the slightly more flexible script
+	python setup.py build_ext --inplace
 	python tests/runtests.py
 else
 # Let setup.py check dependencies, build stuff, and run the test

docs/design.md

@@ -266,3 +266,65 @@ Each table contains:
   from the end of the file will not shorten it; it will only be
   deleted when it has been fully filled and all of the data has been
   subsequently removed.
+
+
+Rocket
+------
+
+Original design had the nilmdb.nilmdb thread (through bulkdata)
+convert from on-disk layout to a Python list, and then the
+nilmdb.server thread (from cherrypy) converts to ASCII.  For at least
+the extraction side of things, it's easy to pass the bulkdata a layout
+name instead, and have it convert directly from on-disk to ASCII
+format, because this conversion can then be shoved into a C module.
+This module, which provides a means for converting directly from
+on-disk format to ASCII or Python lists, is the "rocket" interface.
+Python is still used to manage the files and figure out where the
+data should go; rocket just puts binary data directly in or out of
+those files at specified locations.
+
+Before rocket, testing speed with uint16_6 data, with an end-to-end
+test (extracting data with nilmtool):
+
+- insert: 65 klines/sec
+- extract: 120 klines/sec
+
+After switching to the rocket design, but using the Python version
+(pyrocket):
+
+- insert: 57 klines/sec
+- extract: 120 klines/sec
+
+After switching to a C extension module (rocket.c)
+
+- insert: 74 klines/sec through insert.py; 99.6 klines/sec through nilmtool
+- extract: 335 klines/sec
+
+After client block updates (described below):
+
+- insert: 180 klines/sec through nilmtool (pre-timestamped)
+- extract: 390 klines/sec through nilmtool
+
+Using "insert --timestamp" or "extract --bare" cuts the speed in half.
+
+Blocks versus lines
+-------------------
+
+Generally want to avoid parsing the bulk of the data as lines if
+possible, and transfer things in bigger blocks at once.
+
+Current places where we use lines:
+
+- All data returned by `client.stream_extract`, since it comes from
+  `httpclient.get_gen`, which iterates over lines.  Not sure if this
+  should be changed, because a `nilmtool extract` is just about the
+  same speed as `curl -q .../stream/extract`!
+
+- `client.StreamInserter.insert_iter` and
+  `client.StreamInserter.insert_line`, which should probably get
+  replaced with block versions.  There's no real need to keep
+  updating the timestamp every time we get a new line of data.
+
+  - Finished.  Just a single insert() that takes any length string and
+    does very little processing until it's time to send it to the
+	server.

nilmdb/__init__.py

@@ -1,7 +1,9 @@
 """Main NilmDB import"""
 
-from nilmdb.server import NilmDB, Server
+# These aren't imported automatically, because loading the server
-from nilmdb.client import Client
+# stuff isn't always necessary.
+#from nilmdb.server import NilmDB, Server
+#from nilmdb.client import Client
 
 from nilmdb._version import get_versions
 __version__ = get_versions()['version']

nilmdb/client/client.py

@@ -2,17 +2,15 @@
 
 """Class for performing HTTP client requests via libcurl"""
 
-import nilmdb
 import nilmdb.utils
 import nilmdb.client.httpclient
+from nilmdb.client.errors import ClientError
 
 import time
 import simplejson as json
 import contextlib
 
-def float_to_string(f):
+from nilmdb.utils.time import float_time_to_string
-    """Use repr to maintain full precision in the string output."""
-    return repr(float(f))
 
 def extract_timestamp(line):
     """Extract just the timestamp from a line of data text"""
@@ -21,8 +19,12 @@ def extract_timestamp(line):
 class Client(object):
     """Main client interface to the Nilm database."""
 
-    def __init__(self, url):
+    def __init__(self, url, post_json = False):
-        self.http = nilmdb.client.httpclient.HTTPClient(url)
+        """Initialize client with given URL.  If post_json is true,
+        POST requests are sent with Content-Type 'application/json'
+        instead of the default 'x-www-form-urlencoded'."""
+        self.http = nilmdb.client.httpclient.HTTPClient(url, post_json)
+        self.post_json = post_json
 
     # __enter__/__exit__ allow this class to be a context manager
     def __enter__(self):
@@ -31,8 +33,11 @@ class Client(object):
     def __exit__(self, exc_type, exc_value, traceback):
         self.close()
 
-    def _json_param(self, data):
+    def _json_post_param(self, data):
         """Return compact json-encoded version of parameter"""
+        if self.post_json:
+            # If we're posting as JSON, we don't need to encode it further here
+            return data
         return json.dumps(data, separators=(',',':'))
 
     def close(self):
@@ -52,12 +57,14 @@ class Client(object):
         as a dictionary."""
         return self.http.get("dbinfo")
 
-    def stream_list(self, path = None, layout = None):
+    def stream_list(self, path = None, layout = None, extended = False):
         params = {}
         if path is not None:
             params["path"] = path
         if layout is not None:
             params["layout"] = layout
+        if extended:
+            params["extended"] = 1
         return self.http.get("stream/list", params)
 
     def stream_get_metadata(self, path, keys = None):
@@ -71,7 +78,7 @@ class Client(object):
         metadata."""
         params = {
             "path": path,
-            "data": self._json_param(data)
+            "data": self._json_post_param(data)
             }
         return self.http.post("stream/set_metadata", params)
 
@@ -79,7 +86,7 @@ class Client(object):
         """Update stream metadata from a dictionary"""
         params = {
             "path": path,
-            "data": self._json_param(data)
+            "data": self._json_post_param(data)
             }
         return self.http.post("stream/update_metadata", params)
 
@@ -100,9 +107,9 @@ class Client(object):
             "path": path
         }
         if start is not None:
-            params["start"] = float_to_string(start)
+            params["start"] = float_time_to_string(start)
         if end is not None:
-            params["end"] = float_to_string(end)
+            params["end"] = float_time_to_string(end)
         return self.http.post("stream/remove", params)
 
     @contextlib.contextmanager
@@ -115,36 +122,31 @@ class Client(object):
 
         Example:
           with client.stream_insert_context('/path', start, end) as ctx:
-            ctx.insert_line('1234567890.0 1 2 3 4\\n')
+            ctx.insert('1234567890.0 1 2 3 4\\n')
-            ctx.insert_line('1234567891.0 1 2 3 4\\n')
+            ctx.insert('1234567891.0 1 2 3 4\\n')
 
         For more details, see help for nilmdb.client.client.StreamInserter
 
         This may make multiple requests to the server, if the data is
         large enough or enough time has passed between insertions.
         """
-        ctx = StreamInserter(self, path, start, end)
+        ctx = StreamInserter(self.http, path, start, end)
         yield ctx
         ctx.finalize()
 
     def stream_insert(self, path, data, start = None, end = None):
-        """Insert rows of data into a stream.  data should be an
+        """Insert rows of data into a stream.  data should be a string
-        iterable object that provides ASCII data that matches the
+        or iterable that provides ASCII data that matches the database
-        database layout for path.  See stream_insert_context for
+        layout for path.  See stream_insert_context for details on the
-        details on the 'start' and 'end' parameters."""
+        'start' and 'end' parameters."""
         with self.stream_insert_context(path, start, end) as ctx:
-            ctx.insert_iter(data)
+            if isinstance(data, basestring):
+                ctx.insert(data)
+            else:
+                for chunk in data:
+                    ctx.insert(chunk)
         return ctx.last_response
 
-    def stream_insert_block(self, path, block, start, end):
-        """Insert an entire block of data into a stream.  Like
-        stream_insert, except 'block' contains multiple lines of ASCII
-        text and is sent in one single chunk."""
-        params = { "path": path,
-                   "start": float_to_string(start),
-                   "end": float_to_string(end) }
-        return self.http.put("stream/insert", block, params)
-
     def stream_intervals(self, path, start = None, end = None):
         """
         Return a generator that yields each stream interval.
@@ -153,9 +155,9 @@ class Client(object):
             "path": path
         }
         if start is not None:
-            params["start"] = float_to_string(start)
+            params["start"] = float_time_to_string(start)
         if end is not None:
-            params["end"] = float_to_string(end)
+            params["end"] = float_time_to_string(end)
         return self.http.get_gen("stream/intervals", params)
 
     def stream_extract(self, path, start = None, end = None, count = False):
@@ -171,9 +173,9 @@ class Client(object):
             "path": path,
         }
         if start is not None:
-            params["start"] = float_to_string(start)
+            params["start"] = float_time_to_string(start)
         if end is not None:
-            params["end"] = float_to_string(end)
+            params["end"] = float_time_to_string(end)
         if count:
             params["count"] = 1
         return self.http.get_gen("stream/extract", params)
@@ -193,8 +195,10 @@ class StreamInserter(object):
     The basic data flow is that we are filling a contiguous interval
     on the server, with no gaps, that extends from timestamp 'start'
     to timestamp 'end'.  Data timestamps satisfy 'start <= t < end'.
-    Data is provided by the user one line at a time with
+
-    .insert_line() or .insert_iter().
+    Data is provided to .insert() as ASCII formatted data separated by
+    newlines.  The chunks of data passed to .insert() do not need to
+    match up with the newlines; less or more than one line can be passed.
 
     1. The first inserted line begins a new interval that starts at
     'start'.  If 'start' is not given, it is deduced from the first
@@ -207,7 +211,9 @@ class StreamInserter(object):
     3. The current contiguous interval can be completed by manually
     calling .finalize(), which the context manager will also do
     automatically.  This will send any remaining data to the server,
-    using the 'end' timestamp to end the interval.
+    using the 'end' timestamp to end the interval.  If no 'end'
+    was provided, it is deduced from the last timestamp seen,
+    plus a small delta.
 
     After a .finalize(), inserting new data goes back to step 1.
 
@@ -216,23 +222,20 @@ class StreamInserter(object):
     to change the end time for the interval.
     """
 
-    # See design.md for a discussion of how much data to send.
+    # See design.md for a discussion of how much data to send.  This
-    # These are soft limits -- actual data might be rounded up.
+    # is a soft limit -- we might send up to twice as much or so
-    # We send when we have a certain amount of data queued, or
+    _max_data = 2 * 1024 * 1024
-    # when a certain amount of time has passed since the last send.
-    _max_data = 1048576
-    _max_time = 30
 
     # Delta to add to the final timestamp, if "end" wasn't given
     _end_epsilon = 1e-6
 
-    def __init__(self, client, path, start = None, end = None):
+    def __init__(self, http, path, start = None, end = None):
         """'http' is the httpclient object.  'path' is the database
         path to insert to.  'start' and 'end' are used for the first
         contiguous interval."""
         self.last_response = None
 
-        self._client = client
+        self._http = http
         self._path = path
 
         # Start and end for the overall contiguous interval we're
@@ -240,60 +243,33 @@ class StreamInserter(object):
         self._interval_start = start
         self._interval_end = end
 
-        # Data for the specific block we're building up to send
+        # Current data we're building up to send.  Each string
+        # goes into the array, and gets joined all at once.
         self._block_data = []
         self._block_len = 0
-        self._block_start = None
 
-        # Time of last request
+    def insert(self, data):
-        self._last_time = time.time()
+        """Insert a chunk of ASCII formatted data in string form.  The
+        overall data must consist of lines terminated by '\\n'."""
+        length = len(data)
+        maxdata = self._max_data
 
-        # We keep a buffer of the two most recently inserted lines.
+        if length > maxdata:
-        # Only the older one actually gets processed; the newer one
+            # This could make our buffer more than twice what we
-        # is used to "look-ahead" to the next timestamp if we need
+            # wanted to send, so split it up.  This is a bit
-        # to internally split an insertion into two requests.
+            # inefficient, but the user really shouldn't be providing
-        self._line_old = None
+            # this much data at once.
-        self._line_new = None
+            for cut in range(0, length, maxdata):
-
+                self.insert(data[cut:(cut + maxdata)])
-    def insert_iter(self, iter):
-        """Insert all lines of ASCII formatted data from the given
-        iterable.  Lines must be terminated with '\\n'."""
-        for line in iter:
-            self.insert_line(line)
-
-    def insert_line(self, line, allow_intermediate = True):
-        """Insert a single line of ASCII formatted data.  Line
-        must be terminated with '\\n'."""
-        if line and (len(line) < 1 or line[-1] != '\n'):
-            raise ValueError("lines must end in with a newline character")
-
-        # Store this new line, but process the previous (old) one.
-        # This lets us "look ahead" to the next line.
-        self._line_old = self._line_new
-        self._line_new = line
-        if self._line_old is None:
             return
 
-        # If starting a new block, pull out the timestamp if needed.
+        # Append this string to our list
-        if self._block_start is None:
+        self._block_data.append(data)
-            if self._interval_start is not None:
+        self._block_len += length
-                # User provided a start timestamp.  Use it once, then
-                # clear it for the next block.
-                self._block_start = self._interval_start
-                self._interval_start = None
-            else:
-                # Extract timestamp from the first row
-                self._block_start = extract_timestamp(self._line_old)
 
-        # Save the line
+        # Send the block once we have enough data
-        self._block_data.append(self._line_old)
+        if self._block_len >= maxdata:
-        self._block_len += len(self._line_old)
+            self._send_block(final = False)
-
-        if allow_intermediate:
-            # Send an intermediate block to the server if needed.
-            elapsed = time.time() - self._last_time
-            if (self._block_len > self._max_data) or (elapsed > self._max_time):
-                self._send_block_intermediate()
 
     def update_start(self, start):
         """Update the start time for the next contiguous interval.
@@ -316,63 +292,109 @@ class StreamInserter(object):
 
         If more data is inserted after a finalize(), it will become
         part of a new interval and there may be a gap left in-between."""
-        # Special marker tells insert_line that this is the end
+        self._send_block(final = True)
-        self.insert_line(None, allow_intermediate = False)
 
-        if self._block_len > 0:
+    def _get_first_noncomment(self, block):
-            # We have data pending, so send the final block
+        """Return the (start, end) indices of the first full line in
-            self._send_block_final()
+        block that isn't a comment, or raise IndexError if
-        elif None not in (self._interval_start, self._interval_end):
+        there isn't one."""
-            # We have no data, but enough information to create an
+        start = 0
-            # empty interval.
+        while True:
-            self._block_start = self._interval_start
+            end = block.find('\n', start)
+            if end < 0:
+                raise IndexError
+            if block[start] != '#':
+                return (start, (end + 1))
+            start = end + 1
+
+    def _get_last_noncomment(self, block):
+        """Return the (start, end) indices of the last full line in
+        block[:length] that isn't a comment, or raise IndexError if
+        there isn't one."""
+        end = block.rfind('\n')
+        if end <= 0:
+            raise IndexError
+        while True:
+            start = block.rfind('\n', 0, end)
+            if block[start + 1] != '#':
+                return ((start + 1), end)
+            if start == -1:
+                raise IndexError
+            end = start
+
+    def _send_block(self, final = False):
+        """Send data currently in the block.  The data sent will
+        consist of full lines only, so some might be left over."""
+        # Build the full string to send
+        block = "".join(self._block_data)
+
+        start_ts = self._interval_start
+        if start_ts is None:
+            # Pull start from the first line
+            try:
+                (spos, epos) = self._get_first_noncomment(block)
+                start_ts = extract_timestamp(block[spos:epos])
+            except (ValueError, IndexError):
+                pass # no timestamp is OK, if we have no data
+
+        if final:
+            # For a final block, it must end in a newline, and the
+            # ending timestamp is either the user-provided end,
+            # or the timestamp of the last line plus epsilon.
+            end_ts = self._interval_end
+            try:
+                if block[-1] != '\n':
+                    raise ValueError("final block didn't end with a newline")
+                if end_ts is None:
+                    (spos, epos) = self._get_last_noncomment(block)
+                    end_ts = extract_timestamp(block[spos:epos])
+                    end_ts += self._end_epsilon
+            except (ValueError, IndexError):
+                pass # no timestamp is OK, if we have no data
+            self._block_data = []
+            self._block_len = 0
+
+            # Next block is completely fresh
             self._interval_start = None
-            self._send_block_final()
-        else:
-            # No data, and no timestamps to use to create an empty
-            # interval.
-            pass
-
-        # Make sure both timestamps are emptied for future intervals.
-        self._interval_start = None
-        self._interval_end = None
-
-    def _send_block_intermediate(self):
-        """Send data, when we still have more data to send.
-        Use the timestamp from the next line, so that the blocks
-        are contiguous."""
-        block_end = extract_timestamp(self._line_new)
-        if self._interval_end is not None and block_end > self._interval_end:
-            # Something's fishy -- the timestamp we found is after
-            # the user's specified end.  Limit it here, and the
-            # server will return an error.
-            block_end = self._interval_end
-        self._send_block(block_end)
-
-    def _send_block_final(self):
-        """Send data, when this is the last block for the interval.
-        There is no next line, so figure out the actual interval end
-        using interval_end or end_epsilon."""
-        if self._interval_end is not None:
-            # Use the user's specified end timestamp
-            block_end = self._interval_end
-            # Clear it in case we send more intervals in the future.
             self._interval_end = None
         else:
-            # Add an epsilon to the last timestamp we saw
+            # An intermediate block, e.g. "line1\nline2\nline3\nline4"
-            block_end = extract_timestamp(self._line_old) + self._end_epsilon
+            # We need to save "line3\nline4" for the next block, and
-        self._send_block(block_end)
+            # use the timestamp from "line3" as the ending timestamp
+            # for this one.
+            try:
+                (spos, epos) = self._get_last_noncomment(block)
+                end_ts = extract_timestamp(block[spos:epos])
+            except (ValueError, IndexError):
+                # If we found no timestamp, give up; we'll send this
+                # block later when we have more data.
+                return
+            if spos == 0:
+                # Not enough data to send an intermediate block
+                return
+            if self._interval_end is not None and end_ts > self._interval_end:
+                # User gave us bad endpoints; send it anyway, and let
+                # the server complain so that the error is the same
+                # as if we hadn't done this chunking.
+                end_ts = self._interval_end
+            self._block_data = [ block[spos:] ]
+            self._block_len = (epos - spos)
+            block = block[:spos]
 
-    def _send_block(self, block_end):
+            # Next block continues where this one ended
-        """Send current block to the server"""
+            self._interval_start = end_ts
-        self.last_response = self._client.stream_insert_block(
-            self._path, "".join(self._block_data),
-            self._block_start, block_end)
 
-        # Clear out the block
+        # Double check endpoints
-        self._block_data = []
+        if start_ts is None or end_ts is None:
-        self._block_len = 0
+            # If the block has no non-comment lines, it's OK
-        self._block_start = None
+            try:
+                self._get_first_noncomment(block)
+            except IndexError:
+                return
+            raise ClientError("have data to send, but no start/end times")
 
-        # Note when we sent it
+        # Send it
-        self._last_time = time.time()
+        params = { "path": self._path,
+                   "start": float_time_to_string(start_ts),
+                   "end": float_time_to_string(end_ts) }
+        self.last_response = self._http.put("stream/insert", block, params)

nilmdb/client/httpclient.py

@@ -1,6 +1,5 @@
 """HTTP client library"""
 
-import nilmdb
 import nilmdb.utils
 from nilmdb.client.errors import ClientError, ServerError, Error
 
@@ -10,7 +9,7 @@ import requests
 
 class HTTPClient(object):
     """Class to manage and perform HTTP requests from the client"""
-    def __init__(self, baseurl = ""):
+    def __init__(self, baseurl = "", post_json = False):
         """If baseurl is supplied, all other functions that take
         a URL can be given a relative URL instead."""
         # Verify / clean up URL
@@ -26,6 +25,10 @@ class HTTPClient(object):
         # Saved response, so that tests can verify a few things.
         self._last_response = {}
 
+        # Whether to send application/json POST bodies (versus
+        # x-www-form-urlencoded)
+        self.post_json = post_json
+
     def _handle_error(self, url, code, body):
         # Default variables for exception.  We use the entire body as
         # the default message, in case we can't extract it from a JSON
@@ -57,12 +60,14 @@ class HTTPClient(object):
     def close(self):
         self.session.close()
 
-    def _do_req(self, method, url, query_data, body_data, stream):
+    def _do_req(self, method, url, query_data, body_data, stream, headers):
         url = urlparse.urljoin(self.baseurl, url)
         try:
             response = self.session.request(method, url,
                                             params = query_data,
-                                            data = body_data)
+                                            data = body_data,
+                                            stream = stream,
+                                            headers = headers)
         except requests.RequestException as e:
             raise ServerError(status = "502 Error", url = url,
                               message = str(e.message))
@@ -76,12 +81,13 @@ class HTTPClient(object):
             return (response, False)
 
     # Normal versions that return data directly
-    def _req(self, method, url, query = None, body = None):
+    def _req(self, method, url, query = None, body = None, headers = None):
         """
         Make a request and return the body data as a string or parsed
         JSON object, or raise an error if it contained an error.
         """
-        (response, isjson) = self._do_req(method, url, query, body, False)
+        (response, isjson) = self._do_req(method, url, query, body,
+                                          stream = False, headers = headers)
         if isjson:
             return json.loads(response.content)
         return response.content
@@ -92,24 +98,31 @@ class HTTPClient(object):
 
     def post(self, url, params = None):
         """Simple POST (parameters in body)"""
-        return self._req("POST", url, None, params)
+        if self.post_json:
+            return self._req("POST", url, None,
+                             json.dumps(params),
+                             { 'Content-type': 'application/json' })
+        else:
+            return self._req("POST", url, None, params)
 
     def put(self, url, data, params = None):
         """Simple PUT (parameters in URL, data in body)"""
         return self._req("PUT", url, params, data)
 
     # Generator versions that return data one line at a time.
-    def _req_gen(self, method, url, query = None, body = None):
+    def _req_gen(self, method, url, query = None, body = None, headers = None):
         """
         Make a request and return a generator that gives back strings
         or JSON decoded lines of the body data, or raise an error if
         it contained an eror.
         """
-        (response, isjson) = self._do_req(method, url, query, body, True)
+        (response, isjson) = self._do_req(method, url, query, body,
-        for line in response.iter_lines():
+                                          stream = True, headers = headers)
-            if isjson:
+        if isjson:
+            for line in response.iter_lines():
                 yield json.loads(line)
-            else:
+        else:
+            for line in response.iter_lines():
                 yield line
 
     def get_gen(self, url, params = None):

nilmdb/cmdline/cmdline.py

@@ -1,18 +1,20 @@
 """Command line client functionality"""
 
-import nilmdb
+import nilmdb.client
+
 from nilmdb.utils.printf import *
 from nilmdb.utils import datetime_tz
 import nilmdb.utils.time
 
 import sys
+import os
 import argparse
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
+subcommands = [ "help", "info", "create", "list", "metadata",
-                "remove", "destroy" ]
+                "insert", "extract", "remove", "destroy" ]
 
 # Import the subcommand modules
 subcmd_mods = {}
@@ -29,6 +31,8 @@ class Cmdline(object):
     def __init__(self, argv = None):
         self.argv = argv or sys.argv[1:]
         self.client = None
+        self.def_url = os.environ.get("NILMDB_URL", "http://localhost:12380")
+        self.subcmd = {}
 
     def arg_time(self, toparse):
         """Parse a time string argument"""
@@ -50,18 +54,17 @@ class Cmdline(object):
 
         group = self.parser.add_argument_group("Server")
         group.add_argument("-u", "--url", action="store",
-                           default="http://localhost:12380/",
+                           default=self.def_url,
                            help="NilmDB server URL (default: %(default)s)")
 
-        sub = self.parser.add_subparsers(title="Commands",
+        sub = self.parser.add_subparsers(
-                                         dest="command",
+            title="Commands", dest="command",
-                                         description="Specify --help after "
+            description="Use 'help command' or 'command --help' for more "
-                                         "the command for command-specific "
+            "details on a particular command.")
-                                         "options.")
 
         # Set up subcommands (defined in separate files)
         for cmd in subcommands:
-            subcmd_mods[cmd].setup(self, sub)
+            self.subcmd[cmd] = subcmd_mods[cmd].setup(self, sub)
 
     def die(self, formatstr, *args):
         fprintf(sys.stderr, formatstr + "\n", *args)
@@ -82,13 +85,15 @@ class Cmdline(object):
         if "verify" in self.args:
             self.args.verify(self)
 
-        self.client = nilmdb.Client(self.args.url)
+        self.client = nilmdb.client.Client(self.args.url)
 
-        # Make a test connection to make sure things work
+        # Make a test connection to make sure things work,
-        try:
+        # unless the particular command requests that we don't.
-            server_version = self.client.version()
+        if "no_test_connect" not in self.args:
-        except nilmdb.client.Error as e:
+            try:
-            self.die("error connecting to server: %s", str(e))
+                server_version = self.client.version()
+            except nilmdb.client.Error as e:
+                self.die("error connecting to server: %s", str(e))
 
         # Now dispatch client request to appropriate function.  Parser
         # should have ensured that we don't have any unknown commands

nilmdb/cmdline/create.py

@@ -1,5 +1,4 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
 
 from argparse import RawDescriptionHelpFormatter as raw_form
@@ -26,6 +25,7 @@ Layout types are of the format: type_count
                        help="Path (in database) of new stream, e.g. /foo/bar")
     group.add_argument("layout",
                        help="Layout type for new stream, e.g. float32_8")
+    return cmd
 
 def cmd_create(self):
     """Create new stream"""

nilmdb/cmdline/destroy.py

@@ -1,5 +1,4 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form
@@ -16,6 +15,7 @@ def setup(self, sub):
     group = cmd.add_argument_group("Required arguments")
     group.add_argument("path",
                        help="Path of the stream to delete, e.g. /foo/bar")
+    return cmd
 
 def cmd_destroy(self):
     """Destroy stream"""

nilmdb/cmdline/extract.py

@@ -30,6 +30,7 @@ def setup(self, sub):
                        help="Show raw timestamps in annotated information")
     group.add_argument("-c", "--count", action="store_true",
                        help="Just output a count of matched data points")
+    return cmd
 
 def cmd_extract_verify(self):
     if self.args.start is not None and self.args.end is not None:
@@ -43,7 +44,7 @@ def cmd_extract(self):
     layout = streams[0][1]
 
     if self.args.timestamp_raw:
-        time_string = repr
+        time_string = nilmdb.utils.time.float_time_to_string
     else:
         time_string = nilmdb.utils.time.format_time
 

nilmdb/cmdline/help.py

@@ -0,0 +1,26 @@
+from nilmdb.utils.printf import *
+
+import argparse
+import sys
+
+def setup(self, sub):
+    cmd = sub.add_parser("help", help="Show detailed help for a command",
+                         description="""
+                         Show help for a command. 'help command' is
+                         the same as 'command --help'.
+                         """)
+    cmd.set_defaults(handler = cmd_help)
+    cmd.set_defaults(no_test_connect = True)
+    cmd.add_argument("command", nargs="?",
+                     help="Command to get help about")
+    cmd.add_argument("rest", nargs=argparse.REMAINDER,
+                     help=argparse.SUPPRESS)
+    return cmd
+
+def cmd_help(self):
+    if self.args.command in self.subcmd:
+        self.subcmd[self.args.command].print_help()
+    else:
+        self.parser.print_help()
+
+    return

nilmdb/cmdline/info.py

@@ -1,4 +1,4 @@
-import nilmdb
+import nilmdb.client
 from nilmdb.utils.printf import *
 from nilmdb.utils import human_size
 
@@ -12,6 +12,7 @@ def setup(self, sub):
                          version.
                          """)
     cmd.set_defaults(handler = cmd_info)
+    return cmd
 
 def cmd_info(self):
     """Print info about the server"""

nilmdb/cmdline/insert.py

@@ -1,5 +1,4 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
 import nilmdb.utils.timestamper as timestamper
 import nilmdb.utils.time
@@ -11,42 +10,67 @@ def setup(self, sub):
                          description="""
                          Insert data into a stream.
                          """)
-    cmd.set_defaults(handler = cmd_insert)
+    cmd.set_defaults(verify = cmd_insert_verify,
+                     handler = cmd_insert)
     cmd.add_argument("-q", "--quiet", action='store_true',
                      help='suppress unnecessary messages')
 
     group = cmd.add_argument_group("Timestamping",
                                    description="""
-                                   If timestamps are already provided in the
+                                   To add timestamps, specify the
-                                   input date, use --none.  Otherwise,
+                                   arguments --timestamp and --rate,
-                                   provide --start, or use --filename to
+                                   and provide a starting time.
-                                   try to deduce timestamps from the file.
-
-                                   Set the TZ environment variable to change
-                                   the default timezone.
                                    """)
 
+    group.add_argument("-t", "--timestamp", action="store_true",
+                       help="Add timestamps to each line")
     group.add_argument("-r", "--rate", type=float,
-                       help="""
+                       help="Data rate, in Hz")
-                       If needed, rate in Hz (required when using --start)
+
-                       """)
+    group = cmd.add_argument_group("Start time",
+                                   description="""
+                                   Start time may be manually
+                                   specified with --start, or guessed
+                                   from the filenames using
+                                   --filename.  Set the TZ environment
+                                   variable to change the default
+                                   timezone.""")
+
     exc = group.add_mutually_exclusive_group()
     exc.add_argument("-s", "--start",
                      metavar="TIME", type=self.arg_time,
                      help="Starting timestamp (free-form)")
     exc.add_argument("-f", "--filename", action="store_true",
-                     help="""
+                     help="Use filename to determine start time")
-                     Use filenames to determine start time
+
-                     (default, if filenames are provided)
+    group = cmd.add_argument_group("End time",
-                     """)
+                                   description="""
-    exc.add_argument("-n", "--none", action="store_true",
+                                   End time for the overall stream.
-                     help="Timestamp is already present, don't add one")
+                                   (required when not using --timestamp).
+                                   Set the TZ environment
+                                   variable to change the default
+                                   timezone.""")
+    group.add_argument("-e", "--end",
+                       metavar="TIME", type=self.arg_time,
+                       help="Ending timestamp (free-form)")
 
     group = cmd.add_argument_group("Required parameters")
     group.add_argument("path",
                        help="Path of stream, e.g. /foo/bar")
-    group.add_argument("file", nargs="*", default=['-'],
+    group.add_argument("file", nargs = '?', default='-',
-                       help="File(s) to insert (default: - (stdin))")
+                       help="File to insert (default: - (stdin))")
+    return cmd
+
+def cmd_insert_verify(self):
+    if self.args.timestamp:
+        if not self.args.rate:
+            self.die("error: --rate is needed, but was not specified")
+        if not self.args.filename and self.args.start is None:
+            self.die("error: need --start or --filename when adding timestamps")
+    else:
+        if self.args.start is None or self.args.end is None:
+            self.die("error: when not adding timestamps, --start and "
+                     "--end are required")
 
 def cmd_insert(self):
     # Find requested stream
@@ -54,51 +78,50 @@ def cmd_insert(self):
     if len(streams) != 1:
         self.die("error getting stream info for path %s", self.args.path)
 
-    if self.args.start and len(self.args.file) != 1:
+    arg = self.args
-        self.die("error: --start can only be used with one input file")
 
-    for filename in self.args.file:
+    try:
+        filename = arg.file
         if filename == '-':
             infile = sys.stdin
         else:
             try:
-                infile = open(filename, "r")
+                infile = open(filename, "rb")
             except IOError:
                 self.die("error opening input file %s", filename)
 
-        # Build a timestamper for this file
+        if arg.start is None:
-        if self.args.none:
+            try:
-            ts = timestamper.TimestamperNull(infile)
+                arg.start = nilmdb.utils.time.parse_time(filename).totimestamp()
+            except ValueError:
+                self.die("error extracting start time from filename '%s'",
+                         filename)
+
+        if arg.timestamp:
+            data = timestamper.TimestamperRate(infile, arg.start, arg.rate)
         else:
-            if self.args.start:
+            data = iter(lambda: infile.read(1048576), '')
-                start = self.args.start
-            else:
-                try:
-                    start = nilmdb.utils.time.parse_time(filename)
-                except ValueError:
-                    self.die("error extracting time from filename '%s'",
-                             filename)
-
-            if not self.args.rate:
-                self.die("error: --rate is needed, but was not specified")
-            rate = self.args.rate
-
-            ts = timestamper.TimestamperRate(infile, start, rate)
 
         # Print info
-        if not self.args.quiet:
+        if not arg.quiet:
-            printf("Input file: %s\n", filename)
+            printf(" Input file: %s\n", filename)
-            printf("Timestamper: %s\n", str(ts))
+            printf(" Start time: %s\n",
+                   nilmdb.utils.time.format_time(arg.start))
+            if arg.end:
+                printf("   End time: %s\n",
+                       nilmdb.utils.time.format_time(arg.end))
+            if arg.timestamp:
+                printf("Timestamper: %s\n", str(data))
 
         # Insert the data
-        try:
+        self.client.stream_insert(arg.path, data, arg.start, arg.end)
-            self.client.stream_insert(self.args.path, ts)
+
-        except nilmdb.client.Error as e:
+    except nilmdb.client.Error as e:
-            # TODO: It would be nice to be able to offer better errors
+        # TODO: It would be nice to be able to offer better errors
-            # here, particularly in the case of overlap, which just shows
+        # here, particularly in the case of overlap, which just shows
-            # ugly bracketed ranges of 16-digit numbers and a mangled URL.
+        # ugly bracketed ranges of 16-digit numbers and a mangled URL.
-            # Need to consider adding something like e.prettyprint()
+        # Need to consider adding something like e.prettyprint()
-            # that is smarter about the contents of the error.
+        # that is smarter about the contents of the error.
-            self.die("error inserting data: %s", str(e))
+        self.die("error inserting data: %s", str(e))
 
     return

nilmdb/cmdline/list.py

@@ -24,17 +24,28 @@ def setup(self, sub):
     group.add_argument("-l", "--layout", default="*",
                        help="Match only this stream layout")
 
+    group = cmd.add_argument_group("Interval info")
+    group.add_argument("-E", "--ext", action="store_true",
+                       help="Show extended stream info, like interval "
+                       "extents and row count")
+
     group = cmd.add_argument_group("Interval details")
     group.add_argument("-d", "--detail", action="store_true",
                        help="Show available data time intervals")
-    group.add_argument("-T", "--timestamp-raw", action="store_true",
-                       help="Show raw timestamps in time intervals")
     group.add_argument("-s", "--start",
                        metavar="TIME", type=self.arg_time,
-                       help="Starting timestamp (free-form, inclusive)")
+                       help="Starting timestamp for intervals "
+                       "(free-form, inclusive)")
     group.add_argument("-e", "--end",
                        metavar="TIME", type=self.arg_time,
-                       help="Ending timestamp (free-form, noninclusive)")
+                       help="Ending timestamp for intervals "
+                       "(free-form, noninclusive)")
+
+    group = cmd.add_argument_group("Misc options")
+    group.add_argument("-T", "--timestamp-raw", action="store_true",
+                       help="Show raw timestamps when printing times")
+
+    return cmd
 
 def cmd_list_verify(self):
     # A hidden "path_positional" argument lets the user leave off the
@@ -51,28 +62,41 @@ def cmd_list_verify(self):
         if self.args.start >= self.args.end:
             self.parser.error("start must precede end")
 
+    if self.args.start is not None or self.args.end is not None:
+        if not self.args.detail:
+            self.parser.error("--start and --end only make sense with --detail")
+
 def cmd_list(self):
     """List available streams"""
-    streams = self.client.stream_list()
+    streams = self.client.stream_list(extended = True)
 
     if self.args.timestamp_raw:
-        time_string = repr
+        time_string = nilmdb.utils.time.float_time_to_string
     else:
         time_string = nilmdb.utils.time.format_time
 
-    for (path, layout) in streams:
+    for stream in streams:
+        (path, layout, int_min, int_max, rows, seconds) = stream[:6]
         if not (fnmatch.fnmatch(path, self.args.path) and
                 fnmatch.fnmatch(layout, self.args.layout)):
             continue
 
         printf("%s %s\n", path, layout)
-        if not self.args.detail:
-            continue
 
-        printed = False
+        if self.args.ext:
-        for (start, end) in self.client.stream_intervals(path, self.args.start,
+            if int_min is None or int_max is None:
-                                                         self.args.end):
+                printf("  interval extents: (no data)\n")
-            printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
+            else:
-            printed = True
+                printf("  interval extents: %s -> %s\n",
-        if not printed:
+                       time_string(int_min), time_string(int_max))
-            printf("  (no intervals)\n")
+            printf("        total data: %d rows, %.6f seconds\n",
+                   rows or 0, seconds or 0);
+
+        if self.args.detail:
+            printed = False
+            for (start, end) in self.client.stream_intervals(
+                path, self.args.start, self.args.end):
+                printf("  [ %s -> %s ]\n", time_string(start), time_string(end))
+                printed = True
+            if not printed:
+                printf("  (no intervals)\n")

nilmdb/cmdline/metadata.py

@@ -26,6 +26,7 @@ def setup(self, sub):
     exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
                      help="Update metadata using provided "
                      "key=value pairs")
+    return cmd
 
 def cmd_metadata(self):
     """Manipulate metadata"""

nilmdb/cmdline/remove.py

@@ -1,5 +1,4 @@
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
 
 def setup(self, sub):
@@ -23,6 +22,7 @@ def setup(self, sub):
     group = cmd.add_argument_group("Output format")
     group.add_argument("-c", "--count", action="store_true",
                        help="Output number of data points removed")
+    return cmd
 
 def cmd_remove(self):
     try:

nilmdb/scripts/nilmdb_server.py

@@ -25,6 +25,9 @@ def main():
                        default = os.path.join(os.getcwd(), "db"))
     group.add_argument('-q', '--quiet', help = 'Silence output',
                        action = 'store_true')
+    group.add_argument('-t', '--traceback',
+                       help = 'Provide tracebacks in client errors',
+                       action = 'store_true', default = False)
 
     group = parser.add_argument_group("Debug options")
     group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and '
@@ -35,7 +38,7 @@ def main():
 
     # Create database object.  Needs to be serialized before passing
     # to the Server.
-    db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(args.database)
+    db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database)
 
     # Configure the server
     if args.quiet:
@@ -45,10 +48,12 @@ def main():
     server = nilmdb.server.Server(db,
                                   host = args.address,
                                   port = args.port,
-                                  embedded = embedded)
+                                  embedded = embedded,
+                                  force_traceback = args.traceback)
 
     # Print info
     if not args.quiet:
+        print "Version: %s" % nilmdb.__version__
         print "Database: %s" % (os.path.realpath(args.database))
         if args.address == '0.0.0.0' or args.address == '::':
             host = socket.getfqdn()

nilmdb/server/__init__.py

@@ -16,7 +16,6 @@ try: # pragma: no cover
 except (ImportError, TypeError): # pragma: no cover
     pass
 
-import nilmdb.server.layout
 from nilmdb.server.nilmdb import NilmDB
 from nilmdb.server.server import Server
 from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

nilmdb/server/bulkdata.py

@@ -4,31 +4,24 @@
 # nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
 from __future__ import division
-import nilmdb
 from nilmdb.utils.printf import *
+from nilmdb.utils.time import float_time_to_string as ftts
+import nilmdb.utils
 
 import os
 import cPickle as pickle
-import struct
-import mmap
 import re
+import sys
 
-# If we have the faulthandler module, use it.  All of the mmap stuff
+#from . import pyrocket as rocket
-# might trigger a SIGSEGV or SIGBUS if we're not careful, and
+from . import rocket
-# faulthandler will give a traceback in that case.  (the Python
-# interpreter will still die either way).
-try: # pragma: no cover
-    import faulthandler
-    faulthandler.enable()
-except: # pragma: no cover
-    pass
 
 # Up to 256 open file descriptors at any given time.
 # These variables are global so they can be used in the decorator arguments.
 table_cache_size = 16
 fd_cache_size = 16
 
-@nilmdb.utils.must_close(wrap_verify = True)
+@nilmdb.utils.must_close(wrap_verify = False)
 class BulkData(object):
     def __init__(self, basepath, **kwargs):
         self.basepath = basepath
@@ -83,26 +76,6 @@ class BulkData(object):
             raise ValueError("invalid path; path must contain at least one "
                              "folder")
 
-        # Get layout, and build format string for struct module
-        try:
-            layout = nilmdb.server.layout.get_named(layout_name)
-            struct_fmt = '<d'  # Little endian, double timestamp
-            struct_mapping = {
-                "int8": 'b',
-                "uint8": 'B',
-                "int16": 'h',
-                "uint16": 'H',
-                "int32": 'i',
-                "uint32": 'I',
-                "int64": 'q',
-                "uint64": 'Q',
-                "float32": 'f',
-                "float64": 'd',
-                }
-            struct_fmt += struct_mapping[layout.datatype] * layout.count
-        except KeyError:
-            raise ValueError("no such layout, or bad data types")
-
         # Create the table.  Note that we make a distinction here
         # between NilmDB paths (always Unix style, split apart
         # manually) and OS paths (built up with os.path.join)
@@ -122,11 +95,20 @@ class BulkData(object):
             raise ValueError("subdirs of this path already exist")
         os.mkdir(ospath)
 
-        # Write format string to file
+        try:
-        Table.create(ospath, struct_fmt, self.file_size, self.files_per_dir)
+            # Write format string to file
+            Table.create(ospath, layout_name, self.file_size,
+                         self.files_per_dir)
 
-        # Open and cache it
+            # Open and cache it
-        self.getnode(unicodepath)
+            self.getnode(unicodepath)
+        except:
+            exc_info = sys.exc_info()
+            try:
+                os.rmdir(ospath)
+            except OSError:
+                pass
+            raise exc_info[1], None, exc_info[2]
 
         # Success
         return
@@ -171,53 +153,7 @@ class BulkData(object):
         ospath = os.path.join(self.root, *elements)
         return Table(ospath)
 
-@nilmdb.utils.must_close(wrap_verify = True)
+@nilmdb.utils.must_close(wrap_verify = False)
-class File(object):
-    """Object representing a single file on disk.  Data can be appended,
-    or the self.mmap handle can be used for random reads."""
-
-    def __init__(self, root, subdir, filename):
-        # Create path if it doesn't exist
-        try:
-            os.mkdir(os.path.join(root, subdir))
-        except OSError:
-            pass
-
-        # Open/create file
-        self._f = open(os.path.join(root, subdir, filename), "a+b", 0)
-
-        # Seek to end, and get size
-        self._f.seek(0, 2)
-        self.size = self._f.tell()
-
-        # Open mmap object
-        self.mmap = None
-        self._mmap_reopen()
-
-    def _mmap_reopen(self):
-        if self.size == 0:
-            # Don't mmap if the file is empty; it would fail
-            pass
-        elif self.mmap is None:
-            # Not opened yet, so open it
-            self.mmap = mmap.mmap(self._f.fileno(), 0)
-        else:
-            # Already opened, so just resize it
-            self.mmap.resize(self.size)
-
-    def close(self):
-        if self.mmap is not None:
-            self.mmap.close()
-        self._f.close()
-
-    def append(self, data):
-        # Write data, flush it, and resize our mmap accordingly
-        self._f.write(data)
-        self._f.flush()
-        self.size += len(data)
-        self._mmap_reopen()
-
-@nilmdb.utils.must_close(wrap_verify = True)
 class Table(object):
     """Tools to help access a single table (data at a specific OS path)."""
     # See design.md for design details
@@ -229,19 +165,20 @@ class Table(object):
         return os.path.isfile(os.path.join(root, "_format"))
 
     @classmethod
-    def create(cls, root, struct_fmt, file_size, files_per_dir):
+    def create(cls, root, layout, file_size, files_per_dir):
-        """Initialize a table at the given OS path.
+        """Initialize a table at the given OS path with the
-        'struct_fmt' is a Struct module format description"""
+        given layout string"""
 
         # Calculate rows per file so that each file is approximately
         # file_size bytes.
-        packer = struct.Struct(struct_fmt)
+        rkt = rocket.Rocket(layout, None)
-        rows_per_file = max(file_size // packer.size, 1)
+        rows_per_file = max(file_size // rkt.binary_size, 1)
+        rkt.close()
 
         fmt = { "rows_per_file": rows_per_file,
                 "files_per_dir": files_per_dir,
-                "struct_fmt": struct_fmt,
+                "layout": layout,
-                "version": 1 }
+                "version": 2 }
         with open(os.path.join(root, "_format"), "wb") as f:
             pickle.dump(fmt, f, 2)
 
@@ -250,18 +187,35 @@ class Table(object):
         """'root' is the full OS path to the directory of this table"""
         self.root = root
 
-        # Load the format and build packer
+        # Load the format
         with open(os.path.join(self.root, "_format"), "rb") as f:
             fmt = pickle.load(f)
 
-        if fmt["version"] != 1: # pragma: no cover (just future proofing)
+        if fmt["version"] == 1: # pragma: no cover
-            raise NotImplementedError("version " + fmt["version"] +
+            # We can handle this old version by converting from
+            # struct_fmt back to layout name.
+            compat = { "<dHHHHHH": "uint16_6",
+                       "<dHHHHHHHHH": "uint16_9",
+                       "<dffffffff": "float32_8" }
+            if fmt["struct_fmt"] in compat:
+                fmt["version"] = 2
+                fmt["layout"] = compat[fmt["struct_fmt"]]
+            else:
+                raise NotImplementedError("old version 1 data with format "
+                                          + fmt["struct_fmt"] + " is no good")
+        elif fmt["version"] != 2: # pragma: no cover (just future proofing)
+            raise NotImplementedError("version " + str(fmt["version"]) +
                                       " bulk data store not supported")
 
         self.rows_per_file = fmt["rows_per_file"]
         self.files_per_dir = fmt["files_per_dir"]
-        self.packer = struct.Struct(fmt["struct_fmt"])
+        self.layout = fmt["layout"]
-        self.file_size = self.packer.size * self.rows_per_file
+
+        # Use rocket to get row size and file size
+        rkt = rocket.Rocket(self.layout, None)
+        self.row_size = rkt.binary_size
+        self.file_size = rkt.binary_size * self.rows_per_file
+        rkt.close()
 
         # Find nrows
         self.nrows = self._get_nrows()
@@ -316,26 +270,53 @@ class Table(object):
         # will just get longer but will still sort correctly.
         dirname = sprintf("%04x", filenum // self.files_per_dir)
         filename = sprintf("%04x", filenum % self.files_per_dir)
-        offset = (row % self.rows_per_file) * self.packer.size
+        offset = (row % self.rows_per_file) * self.row_size
         count = self.rows_per_file - (row % self.rows_per_file)
         return (dirname, filename, offset, count)
 
     def _row_from_offset(self, subdir, filename, offset):
         """Return the row number that corresponds to the given
         'subdir/filename' and byte-offset within that file."""
-        if (offset % self.packer.size) != 0: # pragma: no cover; shouldn't occur
+        if (offset % self.row_size) != 0: # pragma: no cover
+            # this shouldn't occur, unless there is some corruption somewhere
             raise ValueError("file offset is not a multiple of data size")
         filenum = int(subdir, 16) * self.files_per_dir + int(filename, 16)
-        row = (filenum * self.rows_per_file) + (offset // self.packer.size)
+        row = (filenum * self.rows_per_file) + (offset // self.row_size)
         return row
 
+    def _remove_or_truncate_file(self, subdir, filename, offset = 0):
+        """Remove the given file, and remove the subdirectory too
+        if it's empty.  If offset is nonzero, truncate the file
+        to that size instead."""
+        # Close potentially open file in file_open LRU cache
+        self.file_open.cache_remove(self, subdir, filename)
+        if offset:
+            # Truncate it
+            with open(os.path.join(self.root, subdir, filename), "r+b") as f:
+                f.truncate(offset)
+        else:
+            # Remove file
+            os.remove(os.path.join(self.root, subdir, filename))
+            # Try deleting subdir, too
+            try:
+                os.rmdir(os.path.join(self.root, subdir))
+            except:
+                pass
+
     # Cache open files
     @nilmdb.utils.lru_cache(size = fd_cache_size,
                             onremove = lambda f: f.close())
     def file_open(self, subdir, filename):
         """Open and map a given 'subdir/filename' (relative to self.root).
         Will be automatically closed when evicted from the cache."""
-        return File(self.root, subdir, filename)
+        # Create path if it doesn't exist
+        try:
+            os.mkdir(os.path.join(self.root, subdir))
+        except OSError:
+            pass
+        # Return a rocket.Rocket object, which contains the open file
+        return rocket.Rocket(self.layout,
+                             os.path.join(self.root, subdir, filename))
 
     def append(self, data):
         """Append the data and flush it to disk.
@@ -351,12 +332,106 @@ class Table(object):
             f = self.file_open(subdir, fname)
 
             # Write the data
-            for i in xrange(count):
+            written = f.append_iter(count, dataiter)
-                row = dataiter.next()
+            if written != count: # pragma: no cover
-                f.append(self.packer.pack(*row))
+                raise Exception("Didn't write the expected number of rows: "
+                                + str(written) + " != " + str(count))
             remaining -= count
             self.nrows += count
 
+    def append_string(self, data, start, end):
+        """Parse the formatted string in 'data', according to the
+        current layout, and append it to the table.  If any timestamps
+        are non-monotonic, or don't fall between 'start' and 'end',
+        a ValueError is raised.
+
+        If this function succeeds, it returns normally.  Otherwise,
+        the table is reverted back to its original state by truncating
+        or deleting files as necessary."""
+        data_offset = 0
+        last_timestamp = -1e12
+        tot_rows = self.nrows
+        count = 0
+        linenum = 0
+        try:
+            while data_offset < len(data):
+                # See how many rows we can fit into the current file,
+                # and open it
+                (subdir, fname, offset, count) = self._offset_from_row(tot_rows)
+                f = self.file_open(subdir, fname)
+
+                # Ask the rocket object to parse and append up to "count"
+                # rows of data, verifying things along the way.
+                try:
+                    (added_rows, data_offset, last_timestamp, linenum
+                     ) = f.append_string(count, data, data_offset, linenum,
+                                         start, end, last_timestamp)
+                except rocket.ParseError as e:
+                    (linenum, errtype, obj) = e.args
+                    if errtype == rocket.ERR_NON_MONOTONIC:
+                        err = sprintf("line %d: timestamp is not monotonically "
+                                      "increasing", linenum)
+                    elif errtype == rocket.ERR_OUT_OF_INTERVAL:
+                        if obj < start:
+                            err = sprintf("line %d: Data timestamp %s < "
+                                          "start time %s", linenum,
+                                          ftts(obj), ftts(start))
+                        else:
+                            err = sprintf("line %d: Data timestamp %s >= "
+                                          "end time %s", linenum,
+                                          ftts(obj), ftts(end))
+                    else:
+                        err = sprintf("line %d: %s", linenum, str(obj))
+                    raise ValueError("error parsing input data: " + err)
+                tot_rows += added_rows
+        except Exception:
+            # Some failure, so try to roll things back by truncating or
+            # deleting files that we may have appended data to.
+            cleanpos = self.nrows
+            while cleanpos <= tot_rows:
+                (subdir, fname, offset, count) = self._offset_from_row(cleanpos)
+                self._remove_or_truncate_file(subdir, fname, offset)
+                cleanpos += count
+            # Re-raise original exception
+            raise
+        else:
+            # Success, so update self.nrows accordingly
+            self.nrows = tot_rows
+
+    def _get_data(self, start, stop, as_string):
+        """Extract data corresponding to Python range [n:m],
+        and returns a numeric list or formatted string,
+        depending on as_string."""
+        if (start is None or
+            stop is None or
+            start > stop or
+            start < 0 or
+            stop > self.nrows):
+            raise IndexError("Index out of range")
+
+        ret = []
+        row = start
+        remaining = stop - start
+        while remaining > 0:
+            (subdir, filename, offset, count) = self._offset_from_row(row)
+            if count > remaining:
+                count = remaining
+            f = self.file_open(subdir, filename)
+            if as_string:
+                ret.append(f.extract_string(offset, count))
+            else:
+                ret.extend(f.extract_list(offset, count))
+            remaining -= count
+            row += count
+        if as_string:
+            return "".join(ret)
+        return ret
+
+    def get_as_text(self, start, stop):
+        """Extract data corresponding to Python range [n:m],
+        and returns a formatted string"""
+        return self._get_data(start, stop, True)
+
     def __getitem__(self, key):
         """Extract data and return it.  Supports simple indexing
         (table[n]) and range slices (table[n:m]).  Returns a nested
@@ -365,41 +440,32 @@ class Table(object):
         # Handle simple slices
         if isinstance(key, slice):
             # Fall back to brute force if the slice isn't simple
-            if ((key.step is not None and key.step != 1) or
+            try:
-                key.start is None or
+                if (key.step is not None and key.step != 1):
-                key.stop is None or
+                    raise IndexError
-                key.start >= key.stop or
+                return self._get_data(key.start, key.stop, False)
-                key.start < 0 or
+            except IndexError:
-                key.stop > self.nrows):
                 return [ self[x] for x in xrange(*key.indices(self.nrows)) ]
 
-            ret = []
+        # Handle single points (inefficiently!)
-            row = key.start
-            remaining = key.stop - key.start
-            while remaining:
-                (subdir, filename, offset, count) = self._offset_from_row(row)
-                if count > remaining:
-                    count = remaining
-                mm = self.file_open(subdir, filename).mmap
-                for i in xrange(count):
-                    ret.append(list(self.packer.unpack_from(mm, offset)))
-                    offset += self.packer.size
-                remaining -= count
-                row += count
-            return ret
-
-        # Handle single points
         if key < 0 or key >= self.nrows:
             raise IndexError("Index out of range")
         (subdir, filename, offset, count) = self._offset_from_row(key)
-        mm = self.file_open(subdir, filename).mmap
+        f = self.file_open(subdir, filename)
-        # unpack_from ignores the mmap object's current seek position
+        return f.extract_list(offset, 1)[0]
-        return list(self.packer.unpack_from(mm, offset))
 
     def _remove_rows(self, subdir, filename, start, stop):
         """Helper to mark specific rows as being removed from a
-        file, and potentially removing or truncating the file itself."""
+        file, and potentially remove or truncate the file itself."""
-        # Import an existing list of deleted rows for this file
+        # Close potentially open file in file_open LRU cache
+        self.file_open.cache_remove(self, subdir, filename)
+
+        # We keep a file like 0000.removed that contains a list of
+        # which rows have been "removed".  Note that we never have to
+        # remove entries from this list, because we never decrease
+        # self.nrows, and so we will never overwrite those locations in the
+        # file.  Only when the list covers the entire extent of the
+        # file will that file be removed.
         datafile = os.path.join(self.root, subdir, filename)
         cachefile = datafile + ".removed"
         try:
@@ -439,20 +505,19 @@ class Table(object):
         # are generally easier if we don't have to special-case that.
         if (len(merged) == 1 and
             merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
-            # Close potentially open file in file_open LRU cache
-            self.file_open.cache_remove(self, subdir, filename)
-
             # Delete files
-            os.remove(datafile)
             if cachefile_present:
                 os.remove(cachefile)
-
+            self._remove_or_truncate_file(subdir, filename, 0)
-            # Try deleting subdir, too
-            try:
-                os.rmdir(os.path.join(self.root, subdir))
-            except:
-                pass
         else:
+            # File needs to stick around.  This means we can get
+            # degenerate cases where we have large files containing as
+            # little as one row.  Try to punch a hole in the file,
+            # so that this region doesn't take up filesystem space.
+            offset = start * self.row_size
+            count = (stop - start) * self.row_size
+            nilmdb.utils.fallocate.punch_hole(datafile, offset, count)
+
             # Update cache.  Try to do it atomically.
             nilmdb.utils.atomic.replace_file(cachefile,
                                              pickle.dumps(merged, 2))
@@ -473,7 +538,7 @@ class Table(object):
             (subdir, filename, offset, count) = self._offset_from_row(row)
             if count > remaining:
                 count = remaining
-            row_offset = offset // self.packer.size
+            row_offset = offset // self.row_size
             # Mark the rows as being removed
             self._remove_rows(subdir, filename, row_offset, row_offset + count)
             remaining -= count

nilmdb/server/interval.pyx

@@ -19,6 +19,8 @@ Intervals are half-open, ie. they include data points with timestamps
 # Fourth version is an optimized rb-tree that stores interval starts
 # and ends directly in the tree, like bxinterval did.
 
+from ..utils.time import float_time_to_string as ftts
+
 cimport rbtree
 cdef extern from "stdint.h":
     ctypedef unsigned long long uint64_t
@@ -47,7 +49,7 @@ cdef class Interval:
         return self.__class__.__name__ + "(" + s + ")"
 
     def __str__(self):
-        return "[" + repr(self.start) + " -> " + repr(self.end) + ")"
+        return "[" + ftts(self.start) + " -> " + ftts(self.end) + ")"
 
     def __cmp__(self, Interval other):
         """Compare two intervals.  If non-equal, order by start then end"""

nilmdb/server/layout.pyx

@@ -4,7 +4,6 @@ import time
 import sys
 import inspect
 import cStringIO
-import numpy as np
 
 cdef enum:
     max_value_count = 64
@@ -42,10 +41,16 @@ class Layout:
 
         if datatype == 'uint16':
             self.parse = self.parse_uint16
-            self.format = self.format_uint16
+            self.format_str = "%.6f" + " %d" * self.count
-        elif datatype == 'float32' or datatype == 'float64':
+            self.format = self.format_generic
+        elif datatype == 'float32':
             self.parse = self.parse_float64
-            self.format = self.format_float64
+            self.format_str = "%.6f" + " %.6e" * self.count
+            self.format = self.format_generic
+        elif datatype == 'float64':
+            self.parse = self.parse_float64
+            self.format_str = "%.6f" + " %.16e" * self.count
+            self.format = self.format_generic
         else:
             raise KeyError("invalid type")
 
@@ -57,15 +62,15 @@ class Layout:
         cdef double ts
         # Return doubles even in float32 case, since they're going into
         # a Python array which would upconvert to double anyway.
-        result = []
+        result = [0] * (self.count + 1)
         cdef char *end
         ts = libc.stdlib.strtod(text, &end)
         if end == text:
             raise ValueError("bad timestamp")
-        result.append(ts)
+        result[0] = ts
         for n in range(self.count):
             text = end
-            result.append(libc.stdlib.strtod(text, &end))
+            result[n+1] = libc.stdlib.strtod(text, &end)
             if end == text:
                 raise ValueError("wrong number of values")
         n = 0
@@ -79,18 +84,18 @@ class Layout:
         cdef int n
         cdef double ts
         cdef int v
-        result = []
         cdef char *end
+        result = [0] * (self.count + 1)
         ts = libc.stdlib.strtod(text, &end)
         if end == text:
             raise ValueError("bad timestamp")
-        result.append(ts)
+        result[0] = ts
         for n in range(self.count):
             text = end
             v = libc.stdlib.strtol(text, &end, 10)
             if v < 0 or v > 65535:
                 raise ValueError("value out of range")
-            result.append(v)
+            result[n+1] = v
             if end == text:
                 raise ValueError("wrong number of values")
         n = 0
@@ -101,25 +106,12 @@ class Layout:
         return (ts, result)
 
     # Formatters
-    def format_float64(self, d):
+    def format_generic(self, d):
         n = len(d) - 1
         if n != self.count:
             raise ValueError("wrong number of values for layout type: "
                              "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
+        return (self.format_str % tuple(d)) + "\n"
-        for i in range(n):
-            s += " %f" % d[i+1]
-        return s + "\n"
-
-    def format_uint16(self, d):
-        n = len(d) - 1
-        if n != self.count:
-            raise ValueError("wrong number of values for layout type: "
-                             "got %d, wanted %d" % (n, self.count))
-        s = "%.6f" % d[0]
-        for i in range(n):
-            s += " %d" % d[i+1]
-        return s + "\n"
 
 # Get a layout by name
 def get_named(typestring):

nilmdb/server/nilmdb.py

@@ -10,7 +10,7 @@ Manages both the SQL database and the table storage backend.
 # Need absolute_import so that "import nilmdb" won't pull in
 # nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
-import nilmdb
+import nilmdb.utils
 from nilmdb.utils.printf import *
 from nilmdb.server.interval import (Interval, DBInterval,
                                     IntervalSet, IntervalError)
@@ -31,11 +31,9 @@ import bisect
 #    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
 # 3: at the end of an explicit transaction, e.g. "with self.con as con:"
 #
-# To speed up testing, or if this transaction speed becomes an issue,
+# To speed things up, we can set 'PRAGMA synchronous=OFF'.  Or, it
-# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF.
+# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL'
-
+# give an equivalent speedup more safely.  That is what is used here.
-
-# Don't touch old entries -- just add new ones.
 _sql_schema_updates = {
     0: """
     -- All streams
@@ -77,7 +75,7 @@ _sql_schema_updates = {
 class NilmDB(object):
     verbose = 0
 
-    def __init__(self, basepath, sync=True, max_results=None,
+    def __init__(self, basepath, max_results=None,
                  bulkdata_args=None):
         if bulkdata_args is None:
             bulkdata_args = {}
@@ -101,10 +99,8 @@ class NilmDB(object):
         self._sql_schema_update()
 
         # See big comment at top about the performance implications of this
-        if sync:
+        self.con.execute("PRAGMA synchronous=NORMAL")
-            self.con.execute("PRAGMA synchronous=FULL")
+        self.con.execute("PRAGMA journal_mode=WAL")
-        else:
-            self.con.execute("PRAGMA synchronous=OFF")
 
         # Approximate largest number of elements that we want to send
         # in a single reply (for stream_intervals, stream_extract)
@@ -269,28 +265,46 @@ class NilmDB(object):
 
         return
 
-    def stream_list(self, path = None, layout = None):
+    def stream_list(self, path = None, layout = None, extended = False):
-        """Return list of [path, layout] lists of all streams
+        """Return list of lists of all streams in the database.
-        in the database.
 
         If path is specified, include only streams with a path that
         matches the given string.
 
         If layout is specified, include only streams with a layout
         that matches the given string.
-        """
-        where = "WHERE 1=1"
-        params = ()
-        if layout:
-            where += " AND layout=?"
-            params += (layout,)
-        if path:
-            where += " AND path=?"
-            params += (path,)
-        result = self.con.execute("SELECT path, layout "
-                                  "FROM streams " + where, params).fetchall()
 
-        return sorted(list(x) for x in result)
+        If extended = False, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extended = True, returns a list of lists containing
+        more information:
+           path
+           layout
+           interval_min (earliest interval start)
+           interval_max (latest interval end)
+           rows         (total number of rows of data)
+           seconds      (total time covered by this stream)
+        """
+        params = ()
+        query = "SELECT streams.path, streams.layout"
+        if extended:
+            query += ", min(ranges.start_time), max(ranges.end_time) "
+            query += ", sum(ranges.end_pos - ranges.start_pos) "
+            query += ", sum(ranges.end_time - ranges.start_time) "
+        query += " FROM streams"
+        if extended:
+            query += " LEFT JOIN ranges ON streams.id = ranges.stream_id"
+        query += " WHERE 1=1"
+        if layout is not None:
+            query += " AND streams.layout=?"
+            params += (layout,)
+        if path is not None:
+            query += " AND streams.path=?"
+            params += (path,)
+        query += " GROUP BY streams.id ORDER BY streams.path"
+        result = self.con.execute(query, params).fetchall()
+        return [ list(x) for x in result ]
 
     def stream_intervals(self, path, start = None, end = None):
         """
@@ -401,8 +415,7 @@ class NilmDB(object):
            path: Path at which to add the data
            start: Starting timestamp
            end: Ending timestamp
-           data: Rows of data, to be passed to bulkdata table.append
+           data: Textual data, formatted according to the layout of path
-                 method.  E.g. nilmdb.layout.Parser.data
            """
         # First check for basic overlap using timestamp info given.
         stream_id = self._stream_id(path)
@@ -412,10 +425,11 @@ class NilmDB(object):
             raise OverlapError("new data overlaps existing data at range: "
                                + str(iset & interval))
 
-        # Insert the data
+        # Tenatively append the data.  This will raise a ValueError if
+        # there are any parse errors.
         table = self.data.getnode(path)
         row_start = table.nrows
-        table.append(data)
+        table.append_string(data, start, end)
         row_end = table.nrows
 
         # Insert the record into the sql database.
@@ -462,9 +476,8 @@ class NilmDB(object):
         """
         Returns (data, restart) tuple.
 
-        data is a list of raw data from the database, suitable for
+        data is ASCII-formatted data from the database, formatted
-        passing to e.g. nilmdb.layout.Formatter to translate into
+        according to the layout of the stream.
-        textual form.
 
         restart, if nonzero, means that there were too many results to
         return in a single request.  The data is complete from the
@@ -505,7 +518,7 @@ class NilmDB(object):
                 restart = table[row_max][0]
 
             # Gather these results up
-            result.extend(table[row_start:row_end])
+            result.append(table.get_as_text(row_start, row_end))
 
             # Count them
             remaining -= row_end - row_start
@@ -515,7 +528,7 @@ class NilmDB(object):
 
         if count:
             return matched
-        return (result, restart)
+        return ("".join(result), restart)
 
     def stream_remove(self, path, start = None, end = None):
         """

nilmdb/server/pyrocket.py

@@ -0,0 +1,143 @@
+# Python implementation of the "rocket" data parsing interface.
+# This interface translates between the binary format on disk
+# and the ASCII format used when communicating with clients.
+
+# This is slow!  Use the C version instead.
+
+from __future__ import absolute_import
+import struct
+import cStringIO
+import itertools
+from . import layout as _layout
+import nilmdb.utils
+from nilmdb.utils.time import float_time_to_string as ftts
+
+ERR_UNKNOWN = 0
+ERR_NON_MONOTONIC = 1
+ERR_OUT_OF_INTERVAL = 2
+class ParseError(Exception):
+    pass
+
+@nilmdb.utils.must_close(wrap_verify = False)
+class Rocket(object):
+    def __init__(self, layout, filename):
+        self.layout = layout
+        if filename:
+            self.file = open(filename, "a+b")
+        else:
+            self.file = None
+
+        # For packing/unpacking into a binary file.
+        # This will change in the C version
+        try:
+            (self.ltype, lcount) = layout.split('_', 2)
+            self.lcount = int(lcount)
+        except:
+            raise ValueError("no such layout: badly formatted string")
+        if self.lcount < 1:
+            raise ValueError("no such layout: bad count")
+        try:
+            struct_fmt = '<d'  # Little endian, double timestamp
+            struct_mapping = {
+                "int8": 'b',
+                "uint8": 'B',
+                "int16": 'h',
+                "uint16": 'H',
+                "int32": 'i',
+                "uint32": 'I',
+                "int64": 'q',
+                "uint64": 'Q',
+                "float32": 'f',
+                "float64": 'd',
+                }
+            struct_fmt += struct_mapping[self.ltype] * self.lcount
+        except KeyError:
+            raise ValueError("no such layout: bad data type")
+        self.packer = struct.Struct(struct_fmt)
+
+        # For packing/unpacking from strings.
+        self.layoutparser = _layout.Layout(self.layout)
+        self.formatter = _layout.Formatter(self.layout)
+
+    def close(self):
+        if self.file:
+            self.file.close()
+
+    @property
+    def binary_size(self):
+        """Return size of one row of data in the binary file, in bytes"""
+        return self.packer.size
+
+    def append_iter(self, maxrows, data):
+        """Append the list data to the file"""
+        # We assume the file is opened in append mode,
+        # so all writes go to the end.
+        written = 0
+        for row in itertools.islice(data, maxrows):
+            self.file.write(self.packer.pack(*row))
+            written += 1
+        self.file.flush()
+        return written
+
+    def append_string(self, count, data, data_offset, linenum,
+                      start, end, last_timestamp):
+        """Parse string and append data.
+
+          count: maximum number of rows to add
+          data: string data
+          data_offset: byte offset into data to start parsing
+          linenum: current line number of data
+          start: starting timestamp for interval
+          end: end timestamp for interval
+          last_timestamp: last timestamp that was previously parsed
+
+        Raises ParseError if timestamps are non-monotonic, outside the
+        start/end interval, etc.
+
+        On success, return a tuple with three values:
+          added_rows: how many rows were added from the file
+          data_offset: current offset into the data string
+          last_timestamp: last timestamp we parsed
+        """
+        # Parse the input data
+        indata = cStringIO.StringIO(data)
+        indata.seek(data_offset)
+        written = 0
+        while written < count:
+            line = indata.readline()
+            linenum += 1
+            if line == "":
+                break
+            comment = line.find('#')
+            if comment >= 0:
+                line = line.split('#', 1)[0]
+            line = line.strip()
+            if line == "":
+                continue
+            try:
+                (ts, row) = self.layoutparser.parse(line)
+            except ValueError as e:
+                raise ParseError(linenum, ERR_UNKNOWN, e)
+            if ts <= last_timestamp:
+                raise ParseError(linenum, ERR_NON_MONOTONIC, ts)
+            last_timestamp = ts
+            if ts < start or ts >= end:
+                raise ParseError(linenum, ERR_OUT_OF_INTERVAL, ts)
+            self.append_iter(1, [row])
+            written += 1
+        return (written, indata.tell(), last_timestamp, linenum)
+
+    def extract_list(self, offset, count):
+        """Extract count rows of data from the file at offset offset.
+        Return a list of lists [[row],[row],...]"""
+        ret = []
+        self.file.seek(offset)
+        for i in xrange(count):
+            data = self.file.read(self.binary_size)
+            ret.append(list(self.packer.unpack(data)))
+        return ret
+
+    def extract_string(self, offset, count):
+        """Extract count rows of data from the file at offset offset.
+        Return an ascii formatted string according to the layout"""
+        return self.formatter.format(self.extract_list(offset, count))

nilmdb/server/rocket.c

@@ -0,0 +1,789 @@
+#include <Python.h>
+#include <structmember.h>
+#include <endian.h>
+
+#include <stdint.h>
+
+/* Values missing from stdint.h */
+#define UINT8_MIN 0
+#define UINT16_MIN 0
+#define UINT32_MIN 0
+#define UINT64_MIN 0
+
+/* Marker values (if min == max, skip range check) */
+#define FLOAT32_MIN 0
+#define FLOAT32_MAX 0
+#define FLOAT64_MIN 0
+#define FLOAT64_MAX 0
+
+/* Somewhat arbitrary, just so we can use fixed sizes for strings
+   etc. */
+static const int MAX_LAYOUT_COUNT = 64;
+
+/* Error object and constants */
+static PyObject *ParseError;
+typedef enum {
+	ERR_OTHER,
+	ERR_NON_MONOTONIC,
+	ERR_OUT_OF_INTERVAL,
+} parseerror_code_t;
+static void add_parseerror_codes(PyObject *module)
+{
+	PyModule_AddIntMacro(module, ERR_OTHER);
+	PyModule_AddIntMacro(module, ERR_NON_MONOTONIC);
+	PyModule_AddIntMacro(module, ERR_OUT_OF_INTERVAL);
+}
+
+/* Helpers to raise ParseErrors.  Use "return raise_str(...)" etc. */
+static PyObject *raise_str(int linenum, int code, const char *string)
+{
+	PyObject *o;
+	o = Py_BuildValue("(iis)", linenum, code, string);
+	if (o != NULL) {
+		PyErr_SetObject(ParseError, o);
+		Py_DECREF(o);
+	}
+	return NULL;
+}
+static PyObject *raise_num(int linenum, int code, double num)
+{
+	PyObject *o;
+	o = Py_BuildValue("(iid)", linenum, code, num);
+	if (o != NULL) {
+		PyErr_SetObject(ParseError, o);
+		Py_DECREF(o);
+	}
+	return NULL;
+}
+
+/****
+ * Layout and type helpers
+ */
+typedef union {
+	int8_t i;
+	uint8_t u;
+} union8_t;
+typedef union {
+	int16_t i;
+	uint16_t u;
+} union16_t;
+typedef union {
+	int32_t i;
+	uint32_t u;
+	float f;
+} union32_t;
+typedef union {
+	int64_t i;
+	uint64_t u;
+	double d;
+} union64_t;
+
+typedef enum {
+	LAYOUT_TYPE_NONE,
+	LAYOUT_TYPE_INT8,
+	LAYOUT_TYPE_UINT8,
+	LAYOUT_TYPE_INT16,
+	LAYOUT_TYPE_UINT16,
+	LAYOUT_TYPE_INT32,
+	LAYOUT_TYPE_UINT32,
+	LAYOUT_TYPE_INT64,
+	LAYOUT_TYPE_UINT64,
+	LAYOUT_TYPE_FLOAT32,
+	LAYOUT_TYPE_FLOAT64,
+} layout_type_t;
+
+struct {
+	char *string;
+	layout_type_t layout;
+	int size;
+} type_lookup[] = {
+	{ "int8",    LAYOUT_TYPE_INT8,    1 },
+	{ "uint8",   LAYOUT_TYPE_UINT8,   1 },
+	{ "int16",   LAYOUT_TYPE_INT16,   2 },
+	{ "uint16",  LAYOUT_TYPE_UINT16,  2 },
+	{ "int32",   LAYOUT_TYPE_INT32,   4 },
+	{ "uint32",  LAYOUT_TYPE_UINT32,  4 },
+	{ "int64",   LAYOUT_TYPE_INT64,   8 },
+	{ "uint64",  LAYOUT_TYPE_UINT64,  8 },
+	{ "float32", LAYOUT_TYPE_FLOAT32, 4 },
+	{ "float64", LAYOUT_TYPE_FLOAT64, 8 },
+	{ NULL }
+};
+
+/****
+ * Object definition, init, etc
+ */
+
+/* Rocket object */
+typedef struct {
+	PyObject_HEAD
+	layout_type_t layout_type;
+	int layout_count;
+	int binary_size;
+	FILE *file;
+	int file_size;
+} Rocket;
+
+/* Dealloc / new */
+static void Rocket_dealloc(Rocket *self)
+{
+	if (self->file) {
+		fprintf(stderr, "rocket: file wasn't closed\n");
+		fclose(self->file);
+		self->file = NULL;
+	}
+	self->ob_type->tp_free((PyObject *)self);
+}
+
+static PyObject *Rocket_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+	Rocket *self;
+
+	self = (Rocket *)type->tp_alloc(type, 0);
+	if (!self)
+		return NULL;
+	self->layout_type = LAYOUT_TYPE_NONE;
+	self->layout_count = 0;
+	self->binary_size = 0;
+	self->file = NULL;
+	self->file_size = -1;
+	return (PyObject *)self;
+}
+
+/* .__init__(layout, file) */
+static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
+{
+	const char *layout, *path;
+	static char *kwlist[] = { "layout", "file", NULL };
+	if (!PyArg_ParseTupleAndKeywords(args, kwds, "sz", kwlist,
+					 &layout, &path))
+		return -1;
+	if (!layout)
+		return -1;
+	if (path) {
+		if ((self->file = fopen(path, "a+b")) == NULL) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return -1;
+		}
+		self->file_size = -1;
+	} else {
+		self->file = NULL;
+	}
+
+	const char *under;
+	char *tmp;
+	under = strchr(layout, '_');
+	if (!under) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"badly formatted string");
+		return -1;
+	}
+	self->layout_count = strtoul(under+1, &tmp, 10);
+	if (self->layout_count < 1 || *tmp != '\0') {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"bad count");
+		return -1;
+	}
+	if (self->layout_count >= MAX_LAYOUT_COUNT) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"count too high");
+		return -1;
+	}
+
+	int i;
+	for (i = 0; type_lookup[i].string; i++)
+		if (strncmp(layout, type_lookup[i].string, under-layout) == 0)
+			break;
+	if (!type_lookup[i].string) {
+		PyErr_SetString(PyExc_ValueError, "no such layout: "
+				"bad data type");
+		return -1;
+	}
+	self->layout_type = type_lookup[i].layout;
+	self->binary_size = 8 + (type_lookup[i].size * self->layout_count);
+
+	return 0;
+}
+
+/* .close() */
+static PyObject *Rocket_close(Rocket *self)
+{
+	if (self->file) {
+		fclose(self->file);
+		self->file = NULL;
+	}
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+/* .file_size property */
+static PyObject *Rocket_get_file_size(Rocket *self)
+{
+	if (!self->file) {
+		PyErr_SetString(PyExc_AttributeError, "no file");
+		return NULL;
+	}
+	if (self->file_size < 0) {
+		int oldpos;
+		if (((oldpos = ftell(self->file)) < 0) ||
+		    (fseek(self->file, 0, SEEK_END) < 0) ||
+		    ((self->file_size = ftell(self->file)) < 0) ||
+		    (fseek(self->file, oldpos, SEEK_SET) < 0)) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return NULL;
+		}
+	}
+	return PyInt_FromLong(self->file_size);
+}
+
+/****
+ * Append from iterator
+ */
+
+/* Helper for writing Python objects to the file */
+static inline void append_pyobject(FILE *out, PyObject *val, layout_type_t type)
+{
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+	int ret = 0;
+
+	switch (type) {
+#define CASE(type, pyconvert, pytype, disktype, htole, bytes)	     \
+		case LAYOUT_TYPE_##type:			     \
+			pytype = pyconvert(val);		     \
+			if (PyErr_Occurred())			     \
+				return;				     \
+			disktype = htole(disktype);		     \
+			ret = fwrite(&disktype, bytes, 1, out);	     \
+			break
+		CASE(INT8,   PyInt_AsLong, t8.i,  t8.u,         , 1);
+		CASE(UINT8,  PyInt_AsLong, t8.u,  t8.u,         , 1);
+		CASE(INT16,  PyInt_AsLong, t16.i, t16.u, htole16, 2);
+		CASE(UINT16, PyInt_AsLong, t16.u, t16.u, htole16, 2);
+		CASE(INT32,  PyInt_AsLong, t32.i, t32.u, htole32, 4);
+		CASE(UINT32, PyInt_AsLong, t32.u, t32.u, htole32, 4);
+		CASE(INT64,  PyInt_AsLong, t64.i, t64.u, htole64, 8);
+		CASE(UINT64, PyInt_AsLong, t64.u, t64.u, htole64, 8);
+		CASE(FLOAT32, PyFloat_AsDouble, t32.f, t32.u, htole32, 4);
+		CASE(FLOAT64, PyFloat_AsDouble, t64.d, t64.u, htole64, 8);
+#undef CASE
+	default:
+		PyErr_SetString(PyExc_TypeError, "unknown type");
+		return;
+	}
+	if (ret <= 0) {
+		PyErr_SetFromErrno(PyExc_OSError);
+	}
+}
+/* .append_iter(maxrows, dataiter) */
+static PyObject *Rocket_append_iter(Rocket *self, PyObject *args)
+{
+	int maxrows;
+	PyObject *iter;
+	PyObject *rowlist;
+	if (!PyArg_ParseTuple(args, "iO:append_iter", &maxrows, &iter))
+		return NULL;
+	if (!PyIter_Check(iter)) {
+		PyErr_SetString(PyExc_TypeError, "need an iterable");
+		return NULL;
+	}
+	if (!self->file) {
+		PyErr_SetString(PyExc_Exception, "no file");
+		return NULL;
+	}
+
+	/* Mark file size so that it will get updated next time it's read */
+	self->file_size = -1;
+
+	int row;
+	for (row = 0; row < maxrows; row++) {
+		rowlist = PyIter_Next(iter);
+		if (!rowlist)
+			break;
+		if (!PyList_Check(rowlist)) {
+			PyErr_SetString(PyExc_TypeError, "rows must be lists");
+			goto row_err;
+		}
+		if (PyList_Size(rowlist) != self->layout_count + 1) {
+			PyErr_SetString(PyExc_TypeError, "short row");
+			goto row_err;
+		}
+
+		/* Extract and write timestamp */
+		append_pyobject(self->file, PyList_GetItem(rowlist, 0),
+				LAYOUT_TYPE_FLOAT64);
+		if (PyErr_Occurred())
+			goto row_err;
+
+		/* Extract and write values */
+		int i;
+		for (i = 0; i < self->layout_count; i++) {
+			append_pyobject(self->file,
+					PyList_GetItem(rowlist, i+1),
+					self->layout_type);
+			if (PyErr_Occurred())
+				goto row_err;
+		}
+	}
+	fflush(self->file);
+	/* All done */
+	return PyLong_FromLong(row);
+row_err:
+	fflush(self->file);
+	Py_DECREF(rowlist);
+	return NULL;
+}
+
+/****
+ * Append from string
+ */
+static inline long int strtol10(const char *nptr, char **endptr) {
+	return strtol(nptr, endptr, 10);
+}
+static inline long int strtoul10(const char *nptr, char **endptr) {
+	return strtoul(nptr, endptr, 10);
+}
+
+/* .append_string(count, data, offset, linenum, start, end, last_timestamp) */
+static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
+{
+	int count;
+	const char *data;
+	int offset;
+	int linenum;
+	double start;
+	double end;
+	double last_timestamp;
+
+	int written = 0;
+	char *endptr;
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+	int i;
+
+	/* It would be nice to use 't#' instead of 's' for data,
+	   but we need the null termination for strto*.  If we had
+	   strnto* that took a length, we could use t# and not require
+	   a copy. */
+	if (!PyArg_ParseTuple(args, "isiiddd:append_string", &count,
+			      &data, &offset, &linenum,
+			      &start, &end, &last_timestamp))
+		return NULL;
+
+	const char *buf = &data[offset];
+	while (written < count && *buf)
+	{
+		linenum++;
+
+		/* Skip leading whitespace and commented lines */
+		while (*buf == ' ' || *buf == '\t')
+			buf++;
+		if (*buf == '#') {
+			while (*buf && *buf != '\n')
+				buf++;
+			if (*buf)
+				buf++;
+			continue;
+		}
+
+		/* Extract timestamp */
+		t64.d = strtod(buf, &endptr);
+		if (endptr == buf)
+			return raise_str(linenum, ERR_OTHER, "bad timestamp");
+		if (t64.d <= last_timestamp)
+			return raise_num(linenum, ERR_NON_MONOTONIC, t64.d);
+		last_timestamp = t64.d;
+		if (t64.d < start || t64.d >= end)
+			return raise_num(linenum, ERR_OUT_OF_INTERVAL, t64.d);
+		t64.u = le64toh(t64.u);
+		if (fwrite(&t64.u, 8, 1, self->file) != 1)
+			goto err;
+		buf = endptr;
+
+		/* Parse all values in the line */
+		switch (self->layout_type) {
+#define CS(type, parsefunc, parsetype, realtype, disktype, letoh, bytes) \
+		case LAYOUT_TYPE_##type:				\
+			/* parse and write in a loop */			\
+			for (i = 0; i < self->layout_count; i++) {	\
+				parsetype = parsefunc(buf, &endptr);	\
+				if (endptr == buf)			\
+					goto wrong_number_of_values;	\
+				if (type##_MIN != type##_MAX &&		\
+				    (parsetype < type##_MIN ||		\
+				     parsetype > type##_MAX))		\
+					goto value_out_of_range;	\
+				realtype = parsetype;			\
+				disktype = letoh(disktype);		\
+				if (fwrite(&disktype, bytes,		\
+					   1, self->file) != 1)		\
+					goto err;			\
+				buf = endptr;				\
+			}						\
+			/* Skip trailing whitespace and comments */	\
+			while (*buf == ' ' || *buf == '\t')		\
+				buf++;					\
+			if (*buf == '#')				\
+				while (*buf && *buf != '\n')		\
+					buf++;				\
+			if (*buf == '\n')				\
+				buf++;					\
+			else if (*buf != '\0')				\
+				goto extra_data_on_line;		\
+			break
+
+			CS(INT8,   strtol10,  t64.i, t8.i,  t8.u,         , 1);
+			CS(UINT8,  strtoul10, t64.u, t8.u,  t8.u,         , 1);
+			CS(INT16,  strtol10,  t64.i, t16.i, t16.u, le16toh, 2);
+			CS(UINT16, strtoul10, t64.u, t16.u, t16.u, le16toh, 2);
+			CS(INT32,  strtol10,  t64.i, t32.i, t32.u, le32toh, 4);
+			CS(UINT32, strtoul10, t64.u, t32.u, t32.u, le32toh, 4);
+			CS(INT64,  strtol10,  t64.i, t64.i, t64.u, le64toh, 8);
+			CS(UINT64, strtoul10, t64.u, t64.u, t64.u, le64toh, 8);
+			CS(FLOAT32, strtod,   t64.d, t32.f, t32.u, le32toh, 4);
+			CS(FLOAT64, strtod,   t64.d, t64.d, t64.u, le64toh, 8);
+#undef CS
+		default:
+			PyErr_SetString(PyExc_TypeError, "unknown type");
+			return NULL;
+		}
+
+		/* Done this line */
+		written++;
+	}
+
+	fflush(self->file);
+
+	/* Build return value and return*/
+	offset = buf - data;
+	PyObject *o;
+	o = Py_BuildValue("(iidi)", written, offset, last_timestamp, linenum);
+	return o;
+err:
+	PyErr_SetFromErrno(PyExc_OSError);
+	return NULL;
+wrong_number_of_values:
+	return raise_str(linenum, ERR_OTHER, "wrong number of values");
+value_out_of_range:
+	return raise_str(linenum, ERR_OTHER, "value out of range");
+extra_data_on_line:
+	return raise_str(linenum, ERR_OTHER, "extra data on line");
+}
+
+/****
+ * Extract to Python list
+ */
+
+static int _extract_handle_params(Rocket *self, PyObject *args, long *count)
+{
+	long offset;
+	if (!PyArg_ParseTuple(args, "ll", &offset, count))
+		return -1;
+	if (!self->file) {
+		PyErr_SetString(PyExc_Exception, "no file");
+		return -1;
+	}
+	/* Seek to target location */
+	if (fseek(self->file, offset, SEEK_SET) < 0) {
+		PyErr_SetFromErrno(PyExc_OSError);
+		return -1;
+	}
+	return 0;
+}
+
+/* Helper for extracting data from a file as a Python object */
+static inline void *extract_pyobject(FILE *in, layout_type_t type)
+{
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+
+	switch (type) {
+#define CASE(type, pyconvert, pytype, disktype, letoh, bytes)		\
+		case LAYOUT_TYPE_##type:				\
+			if (fread(&disktype, bytes, 1, in) <= 0)	\
+				break;					\
+			disktype = letoh(disktype);			\
+			return pyconvert(pytype);			\
+			break
+		CASE(INT8,   PyInt_FromLong, t8.i,  t8.u,         , 1);
+		CASE(UINT8,  PyInt_FromLong, t8.u,  t8.u,         , 1);
+		CASE(INT16,  PyInt_FromLong, t16.i, t16.u, le16toh, 2);
+		CASE(UINT16, PyInt_FromLong, t16.u, t16.u, le16toh, 2);
+		CASE(INT32,  PyInt_FromLong, t32.i, t32.u, le32toh, 4);
+		CASE(UINT32, PyInt_FromLong, t32.u, t32.u, le32toh, 4);
+		CASE(INT64,  PyInt_FromLong, t64.i, t64.u, le64toh, 8);
+		CASE(UINT64, PyInt_FromLong, t64.u, t64.u, le64toh, 8);
+		CASE(FLOAT32, PyFloat_FromDouble, t32.f, t32.u, le32toh, 4);
+		CASE(FLOAT64, PyFloat_FromDouble, t64.d, t64.u, le64toh, 8);
+#undef CASE
+	default:
+		PyErr_SetString(PyExc_TypeError, "unknown type");
+		return NULL;
+	}
+	PyErr_SetString(PyExc_OSError, "failed to read from file");
+	return NULL;
+}
+
+static PyObject *Rocket_extract_list(Rocket *self, PyObject *args)
+{
+	long count;
+	if (_extract_handle_params(self, args, &count) < 0)
+		return NULL;
+
+	/* Make a list to return */
+	PyObject *retlist = PyList_New(0);
+	if (!retlist)
+		return NULL;
+
+	/* Read data into new Python lists */
+	int row;
+	for (row = 0; row < count; row++)
+	{
+		PyObject *rowlist = PyList_New(self->layout_count + 1);
+		if (!rowlist) {
+			Py_DECREF(retlist);
+			return NULL;
+		}
+
+		/* Timestamp */
+		PyObject *entry = extract_pyobject(self->file,
+						   LAYOUT_TYPE_FLOAT64);
+		if (!entry || (PyList_SetItem(rowlist, 0, entry) < 0)) {
+			Py_DECREF(rowlist);
+			Py_DECREF(retlist);
+			return NULL;
+		}
+
+		/* Data */
+		int i;
+		for (i = 0; i < self->layout_count; i++) {
+			PyObject *ent = extract_pyobject(self->file,
+							 self->layout_type);
+			if (!ent || (PyList_SetItem(rowlist, i+1, ent) < 0)) {
+				Py_DECREF(rowlist);
+				Py_DECREF(retlist);
+				return NULL;
+			}
+		}
+
+		/* Add row to return value */
+		if (PyList_Append(retlist, rowlist) < 0) {
+			Py_DECREF(rowlist);
+			Py_DECREF(retlist);
+			return NULL;
+		}
+
+		Py_DECREF(rowlist);
+	}
+	return retlist;
+}
+
+/****
+ * Extract to string
+ */
+
+static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
+{
+	long count;
+	if (_extract_handle_params(self, args, &count) < 0)
+		return NULL;
+
+	char *str = NULL, *new;
+	long len_alloc = 0;
+	long len = 0;
+	int ret;
+
+	/* min space free in string (and the maximum length of one
+	   line); this is generous */
+	const int min_free = 32 * MAX_LAYOUT_COUNT;
+
+	/* how much to allocate at once */
+	const int alloc_size = 1048576;
+
+	int row, i;
+	union8_t t8;
+	union16_t t16;
+	union32_t t32;
+	union64_t t64;
+	for (row = 0; row < count; row++) {
+		/* Make sure there's space for a line */
+		if ((len_alloc - len) < min_free) {
+			/* grow by 1 meg at a time */
+			len_alloc += alloc_size;
+			new = realloc(str, len_alloc);
+			if (new == NULL)
+				goto err;
+			str = new;
+		}
+
+		/* Read and print timestamp */
+		if (fread(&t64.u, 8, 1, self->file) != 1)
+			goto err;
+		t64.u = le64toh(t64.u);
+		/* Timestamps are always printed to the microsecond */
+		ret = sprintf(&str[len], "%.6f", t64.d);
+		if (ret <= 0)
+			goto err;
+		len += ret;
+
+		/* Read and print values */
+		switch (self->layout_type) {
+#define CASE(type, fmt, fmttype, disktype, letoh, bytes)		\
+		case LAYOUT_TYPE_##type:				\
+			/* read and format in a loop */			\
+			for (i = 0; i < self->layout_count; i++) {	\
+				if (fread(&disktype, bytes,		\
+					  1, self->file) < 0)		\
+					goto err;			\
+				disktype = letoh(disktype);		\
+				ret = sprintf(&str[len], " " fmt,	\
+					      fmttype);			\
+				if (ret <= 0)				\
+					goto err;			\
+				len += ret;				\
+			}						\
+			break
+			CASE(INT8,   "%hhd",   t8.i,  t8.u,         , 1);
+			CASE(UINT8,  "%hhu",   t8.u,  t8.u,         , 1);
+			CASE(INT16,  "%hd",    t16.i, t16.u, le16toh, 2);
+			CASE(UINT16, "%hu",    t16.u, t16.u, le16toh, 2);
+			CASE(INT32,  "%d",     t32.i, t32.u, le32toh, 4);
+			CASE(UINT32, "%u",     t32.u, t32.u, le32toh, 4);
+			CASE(INT64,  "%ld",    t64.i, t64.u, le64toh, 8);
+			CASE(UINT64, "%lu",    t64.u, t64.u, le64toh, 8);
+			/* These next two are a bit debatable.  floats
+			   are 6-9 significant figures, so we print 7.
+			   Doubles are 15-19, so we print 17.  This is
+			   similar to the old prep format for float32.
+			*/
+			CASE(FLOAT32, "%.6e",  t32.f, t32.u, le32toh, 4);
+			CASE(FLOAT64, "%.16e", t64.d, t64.u, le64toh, 8);
+#undef CASE
+		default:
+			PyErr_SetString(PyExc_TypeError, "unknown type");
+			if (str) free(str);
+			return NULL;
+		}
+		str[len++] = '\n';
+	}
+
+	PyObject *pystr = PyString_FromStringAndSize(str, len);
+	free(str);
+	return pystr;
+err:
+	if (str) free(str);
+	PyErr_SetFromErrno(PyExc_OSError);
+	return NULL;
+}
+
+/****
+ * Module and type setup
+ */
+
+static PyGetSetDef Rocket_getsetters[] = {
+	{ "file_size", (getter)Rocket_get_file_size, NULL,
+	  "file size in bytes", NULL },
+	{ NULL },
+};
+
+static PyMemberDef Rocket_members[] = {
+	{ "binary_size", T_INT, offsetof(Rocket, binary_size), 0,
+	  "binary size per row" },
+	{ NULL },
+};
+
+static PyMethodDef Rocket_methods[] = {
+	{ "close", (PyCFunction)Rocket_close, METH_NOARGS,
+	  "close(self)\n\n"
+	  "Close file handle" },
+
+	{ "append_iter", (PyCFunction)Rocket_append_iter, METH_VARARGS,
+	  "append_iter(self, maxrows, iterable)\n\n"
+	  "Append up to maxrows of data from iter to the file" },
+
+	{ "append_string", (PyCFunction)Rocket_append_string, METH_VARARGS,
+	  "append_string(self, count, data, offset, line, start, end, ts)\n\n"
+          "Parse string and append data.\n"
+	  "\n"
+	  "  count: maximum number of rows to add\n"
+          "  data: string data\n"
+          "  offset: byte offset into data to start parsing\n"
+          "  line: current line number of data\n"
+          "  start: starting timestamp for interval\n"
+          "  end: end timestamp for interval\n"
+          "  ts: last timestamp that was previously parsed\n"
+	  "\n"
+	  "Raises ParseError if timestamps are non-monotonic, outside\n"
+	  "the start/end interval etc.\n"
+	  "\n"
+          "On success, return a tuple with three values:\n"
+          "  added_rows: how many rows were added from the file\n"
+          "  data_offset: current offset into the data string\n"
+          "  last_timestamp: last timestamp we parsed" },
+
+	{ "extract_list", (PyCFunction)Rocket_extract_list, METH_VARARGS,
+	  "extract_list(self, offset, count)\n\n"
+	  "Extract count rows of data from the file at offset offset.\n"
+	  "Return a list of lists [[row],[row],...]" },
+
+	{ "extract_string", (PyCFunction)Rocket_extract_string, METH_VARARGS,
+	  "extract_string(self, offset, count)\n\n"
+	  "Extract count rows of data from the file at offset offset.\n"
+	  "Return an ascii formatted string according to the layout" },
+
+	{ NULL },
+};
+
+static PyTypeObject RocketType = {
+	PyObject_HEAD_INIT(NULL)
+
+	.tp_name	= "rocket.Rocket",
+	.tp_basicsize	= sizeof(Rocket),
+	.tp_flags	= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+
+	.tp_new		= Rocket_new,
+	.tp_dealloc	= (destructor)Rocket_dealloc,
+	.tp_init	= (initproc)Rocket_init,
+	.tp_methods	= Rocket_methods,
+	.tp_members	= Rocket_members,
+	.tp_getset	= Rocket_getsetters,
+
+	.tp_doc		= ("rocket.Rocket(layout, file)\n\n"
+			   "C implementation of the \"rocket\" data parsing\n"
+			   "interface, which translates between the binary\n"
+			   "format on disk and the ASCII or Python list\n"
+			   "format used when communicating with the rest of\n"
+			   "the system.")
+};
+
+static PyMethodDef module_methods[] = {
+	{ NULL },
+};
+
+PyMODINIT_FUNC
+initrocket(void)
+{
+	PyObject *module;
+
+	RocketType.tp_new = PyType_GenericNew;
+	if (PyType_Ready(&RocketType) < 0)
+		return;
+
+	module = Py_InitModule3("rocket", module_methods,
+				"Rocket data parsing and formatting module");
+	Py_INCREF(&RocketType);
+	PyModule_AddObject(module, "Rocket", (PyObject *)&RocketType);
+
+	ParseError = PyErr_NewException("rocket.ParseError", NULL, NULL);
+	Py_INCREF(ParseError);
+	PyModule_AddObject(module, "ParseError", ParseError);
+	add_parseerror_codes(module);
+
+	return;
+}

nilmdb/server/server.py

@@ -3,7 +3,7 @@
 # Need absolute_import so that "import nilmdb" won't pull in
 # nilmdb.py, but will pull the nilmdb module instead.
 from __future__ import absolute_import
-import nilmdb
+import nilmdb.server
 from nilmdb.utils.printf import *
 from nilmdb.server.errors import NilmDBError
 
@@ -12,7 +12,6 @@ import sys
 import os
 import simplejson as json
 import decorator
-import traceback
 import psutil
 
 class NilmApp(object):
@@ -45,12 +44,17 @@ def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
     bug #1200.  This throws them as generic Exceptions instead so that
     they make it through.
     """
+    exc_info = None
     try:
         for val in func(*args, **kwargs):
             yield val
     except (LookupError, UnicodeError):
-        raise Exception("bug workaround; real exception is:\n" +
+        # Re-raise it, but maintain the original traceback
-                        traceback.format_exc())
+        exc_info = sys.exc_info()
+        new_exc = Exception(exc_info[0].__name__ + ": " + str(exc_info[1]))
+        raise new_exc, None, exc_info[2]
+    finally:
+        del exc_info
 
 def exception_to_httperror(*expected):
     """Return a decorator-generating function that catches expected
@@ -61,26 +65,74 @@ def exception_to_httperror(*expected):
             pass
     """
     def wrapper(func, *args, **kwargs):
+        exc_info = None
         try:
             return func(*args, **kwargs)
-        except expected as e:
+        except expected:
-            message = sprintf("%s", str(e))
+            # Re-raise it, but maintain the original traceback
-            raise cherrypy.HTTPError("400 Bad Request", message)
+            exc_info = sys.exc_info()
+            new_exc = cherrypy.HTTPError("400 Bad Request", str(exc_info[1]))
+            raise new_exc, None, exc_info[2]
+        finally:
+            del exc_info
     # We need to preserve the function's argspecs for CherryPy to
     # handle argument errors correctly.  Decorator.decorator takes
     # care of that.
     return decorator.decorator(wrapper)
 
-# Custom Cherrypy tools
+# Custom CherryPy tools
-def allow_methods(methods):
+
-    method = cherrypy.request.method.upper()
+def CORS_allow(methods):
-    if method not in methods:
+    """This does several things:
-        if method in cherrypy.request.methods_with_bodies:
+
-            cherrypy.request.body.read()
+    Handles CORS preflight requests.
-        allowed = ', '.join(methods)
+    Adds Allow: header to all requests.
-        cherrypy.response.headers['Allow'] = allowed
+    Raise 405 if request.method not in method.
-        raise cherrypy.HTTPError(405, method + " not allowed; use " + allowed)
+
-cherrypy.tools.allow_methods = cherrypy.Tool('before_handler', allow_methods)
+    It is similar to cherrypy.tools.allow, with the CORS stuff added.
+    """
+    request = cherrypy.request.headers
+    response = cherrypy.response.headers
+
+    if not isinstance(methods, (tuple, list)): # pragma: no cover
+        methods = [ methods ]
+    methods = [ m.upper() for m in methods if m ]
+    if not methods: # pragma: no cover
+        methods = [ 'GET', 'HEAD' ]
+    elif 'GET' in methods and 'HEAD' not in methods: # pragma: no cover
+        methods.append('HEAD')
+    response['Allow'] = ', '.join(methods)
+
+    # Allow all origins
+    if 'Origin' in request:
+        response['Access-Control-Allow-Origin'] = request['Origin']
+
+    # If it's a CORS request, send response.
+    request_method = request.get("Access-Control-Request-Method", None)
+    request_headers = request.get("Access-Control-Request-Headers", None)
+    if (cherrypy.request.method == "OPTIONS" and
+        request_method and request_headers):
+        response['Access-Control-Allow-Headers'] = request_headers
+        response['Access-Control-Allow-Methods'] = ', '.join(methods)
+        # Try to stop further processing and return a 200 OK
+        cherrypy.response.status = "200 OK"
+        cherrypy.response.body = ""
+        cherrypy.request.handler = lambda: ""
+        return
+
+    # Reject methods that were not explicitly allowed
+    if cherrypy.request.method not in methods:
+        raise cherrypy.HTTPError(405)
+
+cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
+
+# Helper for json_in tool to process JSON data into normal request
+# parameters.
+def json_to_request_params(body):
+    cherrypy.lib.jsontools.json_processor(body)
+    if not isinstance(cherrypy.request.json, dict):
+        raise cherrypy.HTTPError(415)
+    cherrypy.request.params.update(cherrypy.request.json)
 
 # CherryPy apps
 class Root(NilmApp):
@@ -120,21 +172,30 @@ class Stream(NilmApp):
     """Stream-specific operations"""
 
     # /stream/list
-    # /stream/list?layout=PrepData
+    # /stream/list?layout=float32_8
-    # /stream/list?path=/newton/prep
+    # /stream/list?path=/newton/prep&extended=1
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def list(self, path = None, layout = None):
+    def list(self, path = None, layout = None, extended = None):
         """List all streams in the database.  With optional path or
         layout parameter, just list streams that match the given path
-        or layout"""
+        or layout.
-        return self.db.stream_list(path, layout)
 
-    # /stream/create?path=/newton/prep&layout=PrepData
+        If extent is not given, returns a list of lists containing
+        the path and layout: [ path, layout ]
+
+        If extended is provided, returns a list of lists containing
+        extended info: [ path, layout, extent_min, extent_max,
+        total_rows, total_seconds ].  More data may be added.
+        """
+        return self.db.stream_list(path, layout, bool(extended))
+
+    # /stream/create?path=/newton/prep&layout=float32_8
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError, ValueError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def create(self, path, layout):
         """Create a new stream in the database.  Provide path
         and one of the nilmdb.layout.layouts keys.
@@ -143,9 +204,10 @@ class Stream(NilmApp):
 
     # /stream/destroy?path=/newton/prep
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def destroy(self, path):
         """Delete a stream and its associated data."""
         return self.db.stream_destroy(path)
@@ -174,33 +236,47 @@ class Stream(NilmApp):
                 result[k] = None
         return result
 
+    # Helper for set_metadata and get_metadata
+    def _metadata_helper(self, function, path, data):
+        if not isinstance(data, dict):
+            try:
+                data = dict(json.loads(data))
+            except TypeError as e:
+                raise NilmDBError("can't parse 'data' parameter: " + e.message)
+        for key in data:
+            if not (isinstance(data[key], basestring) or
+                    isinstance(data[key], float) or
+                    isinstance(data[key], int)):
+                raise NilmDBError("metadata values must be a string or number")
+        function(path, data)
+
     # /stream/set_metadata?path=/newton/prep&data=<json>
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError, TypeError)
+    @exception_to_httperror(NilmDBError, LookupError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def set_metadata(self, path, data):
-        """Set metadata for the named stream, replacing any
+        """Set metadata for the named stream, replacing any existing
-        existing metadata.  Data should be a json-encoded
+        metadata.  Data can be json-encoded or a plain dictionary."""
-        dictionary"""
+        self._metadata_helper(self.db.stream_set_metadata, path, data)
-        data_dict = json.loads(data)
-        self.db.stream_set_metadata(path, data_dict)
 
     # /stream/update_metadata?path=/newton/prep&data=<json>
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    @exception_to_httperror(NilmDBError, LookupError, TypeError)
+    @exception_to_httperror(NilmDBError, LookupError, ValueError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def update_metadata(self, path, data):
-        """Update metadata for the named stream.  Data
+        """Set metadata for the named stream, replacing any existing
-        should be a json-encoded dictionary"""
+        metadata.  Data can be json-encoded or a plain dictionary."""
-        data_dict = json.loads(data)
+        self._metadata_helper(self.db.stream_update_metadata, path, data)
-        self.db.stream_update_metadata(path, data_dict)
 
     # /stream/insert?path=/newton/prep
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    @cherrypy.tools.allow_methods(methods = ["PUT"])
+    @exception_to_httperror(NilmDBError, ValueError)
+    @cherrypy.tools.CORS_allow(methods = ["PUT"])
     def insert(self, path, start, end):
         """
         Insert new data into the database.  Provide textual data
@@ -216,16 +292,6 @@ class Stream(NilmApp):
         streams = self.db.stream_list(path = path)
         if len(streams) != 1:
             raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
-
-        # Parse the input data
-        try:
-            parser = nilmdb.server.layout.Parser(layout)
-            parser.parse(body)
-        except nilmdb.server.layout.ParserError as e:
-            raise cherrypy.HTTPError("400 Bad Request",
-                                     "error parsing input data: " +
-                                     e.message)
 
         # Check limits
         start = float(start)
@@ -233,20 +299,10 @@ class Stream(NilmApp):
         if start >= end:
             raise cherrypy.HTTPError("400 Bad Request",
                                      "start must precede end")
-        if parser.min_timestamp is not None and parser.min_timestamp < start:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.min_timestamp) +
-                                     " < start time " + repr(start))
-        if parser.max_timestamp is not None and parser.max_timestamp >= end:
-            raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
-                                     repr(parser.max_timestamp) +
-                                     " >= end time " + repr(end))
 
-        # Now do the nilmdb insert, passing it the parser full of data.
+        # Pass the data directly to nilmdb, which will parse it and
-        try:
+        # raise a ValueError if there are any problems.
-            self.db.stream_insert(path, start, end, parser.data)
+        self.db.stream_insert(path, start, end, body)
-        except NilmDBError as e:
-            raise cherrypy.HTTPError("400 Bad Request", e.message)
 
         # Done
         return
@@ -254,9 +310,10 @@ class Stream(NilmApp):
     # /stream/remove?path=/newton/prep
     # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
     @cherrypy.expose
+    @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(NilmDBError)
-    @cherrypy.tools.allow_methods(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods = ["POST"])
     def remove(self, path, start = None, end = None):
         """
         Remove data from the backend database.  Removes all data in
@@ -342,10 +399,6 @@ class Stream(NilmApp):
         streams = self.db.stream_list(path = path)
         if len(streams) != 1:
             raise cherrypy.HTTPError("404 Not Found", "No such stream")
-        layout = streams[0][1]
-
-        # Get formatter
-        formatter = nilmdb.server.layout.Formatter(layout)
 
         @workaround_cp_bug_1200
         def content(start, end, count):
@@ -357,9 +410,7 @@ class Stream(NilmApp):
 
             while True:
                 (data, restart) = self.db.stream_extract(path, start, end)
-
+                yield data
-                # Format the data and yield it
-                yield formatter.format(data)
 
                 if restart == 0:
                     return
@@ -400,7 +451,7 @@ class Server(object):
             'server.socket_host': host,
             'server.socket_port': port,
             'engine.autoreload_on': False,
-            'server.max_request_body_size': 4*1024*1024,
+            'server.max_request_body_size': 8*1024*1024,
             })
         if self.embedded:
             cherrypy.config.update({ 'environment': 'embedded' })
@@ -411,16 +462,20 @@ class Server(object):
             'error_page.default': self.json_error_page,
             })
 
-        # Send a permissive Access-Control-Allow-Origin (CORS) header
+        # Some default headers to just help identify that things are working
-        # with all responses so that browsers can send cross-domain
+        app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
-        # requests to this server.
-        app_config.update({ 'response.headers.Access-Control-Allow-Origin':
-                            '*' })
 
-        # Only allow GET and HEAD by default.  Individual handlers
+        # Set up Cross-Origin Resource Sharing (CORS) handler so we
-        # can override.
+        # can correctly respond to browsers' CORS preflight requests.
-        app_config.update({ 'tools.allow_methods.on': True,
+        # This also limits verbs to GET and HEAD by default.
-                            'tools.allow_methods.methods': ['GET', 'HEAD'] })
+        app_config.update({ 'tools.CORS_allow.on': True,
+                            'tools.CORS_allow.methods': ['GET', 'HEAD'] })
+
+        # Configure the 'json_in' tool to also allow other content-types
+        # (like x-www-form-urlencoded), and to treat JSON as a dict that
+        # fills requests.param.
+        app_config.update({ 'tools.json_in.force': False,
+                            'tools.json_in.processor': json_to_request_params })
 
         # Send tracebacks in error responses.  They're hidden by the
         # error_page function for client errors (code 400-499).

nilmdb/utils/__init__.py

@@ -8,3 +8,5 @@ from nilmdb.utils.diskusage import du, human_size
 from nilmdb.utils.mustclose import must_close
 from nilmdb.utils import atomic
 import nilmdb.utils.threadsafety
+import nilmdb.utils.fallocate
+import nilmdb.utils.time

nilmdb/utils/fallocate.py

@@ -0,0 +1,49 @@
+# Implementation of hole punching via fallocate, if the OS
+# and filesystem support it.
+
+try:
+    import os
+    import ctypes
+    import ctypes.util
+
+    def make_fallocate():
+        libc_name = ctypes.util.find_library('c')
+        libc = ctypes.CDLL(libc_name, use_errno=True)
+
+        _fallocate = libc.fallocate
+        _fallocate.restype = ctypes.c_int
+        _fallocate.argtypes = [ ctypes.c_int, ctypes.c_int,
+                                ctypes.c_int64, ctypes.c_int64 ]
+
+        del libc
+        del libc_name
+
+        def fallocate(fd, mode, offset, len_):
+            res = _fallocate(fd, mode, offset, len_)
+            if res != 0: # pragma: no cover
+                errno = ctypes.get_errno()
+                raise IOError(errno, os.strerror(errno))
+        return fallocate
+
+    fallocate = make_fallocate()
+    del make_fallocate
+except Exception: # pragma: no cover
+    fallocate = None
+
+FALLOC_FL_KEEP_SIZE = 0x01
+FALLOC_FL_PUNCH_HOLE = 0x02
+
+def punch_hole(filename, offset, length, ignore_errors = True):
+    """Punch a hole in the file.  This isn't well supported, so errors
+    are ignored by default."""
+    try:
+        if fallocate is None: # pragma: no cover
+            raise IOError("fallocate not available")
+        with open(filename, "r+") as f:
+            fallocate(f.fileno(),
+                      FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+                      offset, length)
+    except IOError: # pragma: no cover
+        if ignore_errors:
+            return
+        raise

nilmdb/utils/time.py

@@ -19,6 +19,16 @@ def parse_time(toparse):
     except ValueError:
         pass
 
+    # Try to treat it as a single double
+    try:
+        timestamp = float(toparse)
+        # range is from about year 2001 - 2065
+        if timestamp < 1e9 or timestamp > 3e9:
+            raise ValueError
+        return datetime_tz.datetime_tz.fromtimestamp(timestamp)
+    except ValueError:
+        pass
+
     # Try to extract a substring in a condensed format that we expect
     # to see in a filename or header comment
     res = re.search(r"(^|[^\d])("            # non-numeric or SOL
@@ -52,3 +62,8 @@ def format_time(timestamp):
     """
     dt = datetime_tz.datetime_tz.fromtimestamp(timestamp)
     return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
+
+def float_time_to_string(timestamp):
+    """Convert a floating-point Unix timestamp to a string,
+    like '1234567890.000000'"""
+    return "%.6f" % timestamp

nilmdb/utils/timestamper.py

@@ -70,9 +70,6 @@ class TimestamperRate(Timestamper):
                     raise StopIteration
                 yield sprintf("%.6f ", start + n / rate)
                 n += 1
-        # Handle case where we're passed a datetime or datetime_tz object
-        if "totimestamp" in dir(start):
-            start = start.totimestamp()
         Timestamper.__init__(self, infile, iterator(start, rate, end))
         self.start = start
         self.rate = rate
@@ -92,13 +89,3 @@ class TimestamperNow(Timestamper):
         Timestamper.__init__(self, infile, iterator())
     def __str__(self):
         return "TimestamperNow(...)"
-
-class TimestamperNull(Timestamper):
-    """Timestamper that adds nothing to each line"""
-    def __init__(self, infile):
-        def iterator():
-            while True:
-                yield ""
-        Timestamper.__init__(self, infile, iterator())
-    def __str__(self):
-        return "TimestamperNull(...)"

setup.py

@@ -56,7 +56,7 @@ try:
 except ImportError:
     use_cython = False
 
-ext_modules = []
+ext_modules = [ Extension('nilmdb.server.rocket', ['nilmdb/server/rocket.c' ]) ]
 for modulename in cython_modules:
     filename = modulename.replace('.','/')
     if use_cython:

tests/data/extract-1

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
 # start: Fri, 23 Mar 2012 10:00:30.000000 +0000
 # end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+1332496830.000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830.008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830.016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+1332496830.025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+1332496830.033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+1332496830.041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+1332496830.050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+1332496830.058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+1332496830.066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+1332496830.075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+1332496830.083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+1332496830.091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+1332496830.100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+1332496830.108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+1332496830.116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+1332496830.125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+1332496830.133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+1332496830.141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+1332496830.150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+1332496830.158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+1332496830.166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+1332496830.175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+1332496830.183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+1332496830.191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+1332496830.200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+1332496830.208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+1332496830.216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+1332496830.225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+1332496830.233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+1332496830.241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+1332496830.250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+1332496830.258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+1332496830.266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+1332496830.275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+1332496830.283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+1332496830.291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+1332496830.300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+1332496830.308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+1332496830.316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+1332496830.325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+1332496830.333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+1332496830.341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+1332496830.350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+1332496830.358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+1332496830.366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+1332496830.375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+1332496830.383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+1332496830.391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+1332496830.400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+1332496830.408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+1332496830.416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+1332496830.425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+1332496830.433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+1332496830.441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+1332496830.450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+1332496830.458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+1332496830.466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+1332496830.475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+1332496830.483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+1332496830.491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+1332496830.500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+1332496830.508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+1332496830.516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+1332496830.525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+1332496830.533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+1332496830.541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+1332496830.550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+1332496830.558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+1332496830.566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+1332496830.575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+1332496830.583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+1332496830.591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+1332496830.600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+1332496830.608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+1332496830.616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+1332496830.625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+1332496830.633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+1332496830.641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+1332496830.650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+1332496830.658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+1332496830.666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+1332496830.675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+1332496830.683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+1332496830.691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+1332496830.700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+1332496830.708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+1332496830.716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+1332496830.725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+1332496830.733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+1332496830.741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+1332496830.750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+1332496830.758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+1332496830.766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+1332496830.775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+1332496830.783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+1332496830.791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+1332496830.800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+1332496830.808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+1332496830.816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+1332496830.825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+1332496830.833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+1332496830.841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+1332496830.850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+1332496830.858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+1332496830.866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+1332496830.875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+1332496830.883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+1332496830.891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+1332496830.900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+1332496830.908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+1332496830.916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+1332496830.925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+1332496830.933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+1332496830.941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+1332496830.950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+1332496830.958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+1332496830.966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+1332496830.975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+1332496830.983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830.991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-2

@@ -1,119 +1,119 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830.008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830.016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+1332496830.025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+1332496830.033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+1332496830.041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+1332496830.050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+1332496830.058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+1332496830.066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+1332496830.075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+1332496830.083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+1332496830.091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+1332496830.100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+1332496830.108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+1332496830.116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+1332496830.125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+1332496830.133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+1332496830.141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+1332496830.150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+1332496830.158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+1332496830.166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+1332496830.175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+1332496830.183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+1332496830.191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+1332496830.200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+1332496830.208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+1332496830.216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+1332496830.225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+1332496830.233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+1332496830.241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+1332496830.250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+1332496830.258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+1332496830.266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+1332496830.275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+1332496830.283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+1332496830.291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+1332496830.300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+1332496830.308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+1332496830.316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+1332496830.325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+1332496830.333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+1332496830.341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+1332496830.350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+1332496830.358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+1332496830.366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+1332496830.375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+1332496830.383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+1332496830.391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+1332496830.400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+1332496830.408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+1332496830.416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+1332496830.425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+1332496830.433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+1332496830.441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+1332496830.450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+1332496830.458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+1332496830.466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+1332496830.475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+1332496830.483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+1332496830.491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+1332496830.500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+1332496830.508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+1332496830.516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+1332496830.525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+1332496830.533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+1332496830.541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+1332496830.550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+1332496830.558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+1332496830.566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+1332496830.575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+1332496830.583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+1332496830.591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+1332496830.600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+1332496830.608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+1332496830.616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+1332496830.625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+1332496830.633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+1332496830.641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+1332496830.650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+1332496830.658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+1332496830.666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+1332496830.675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+1332496830.683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+1332496830.691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+1332496830.700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+1332496830.708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+1332496830.716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+1332496830.725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+1332496830.733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+1332496830.741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+1332496830.750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+1332496830.758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+1332496830.766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+1332496830.775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+1332496830.783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+1332496830.791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+1332496830.800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+1332496830.808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+1332496830.816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+1332496830.825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+1332496830.833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+1332496830.841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+1332496830.850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+1332496830.858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+1332496830.866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+1332496830.875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+1332496830.883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+1332496830.891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+1332496830.900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+1332496830.908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+1332496830.916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+1332496830.925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+1332496830.933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+1332496830.941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+1332496830.950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+1332496830.958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+1332496830.966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+1332496830.975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+1332496830.983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830.991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-3

@@ -1 +1 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830.008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03

tests/data/extract-4

@@ -1,2 +1,2 @@
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830.008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830.016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03

tests/data/extract-5

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
 # start: Fri, 23 Mar 2012 10:00:30.000000 +0000
 # end: Fri, 23 Mar 2012 10:00:31.000000 +0000
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-6

@@ -1,120 +1,120 @@
-251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/extract-7

@@ -1,124 +1,124 @@
 # path: /newton/prep
-# layout: PrepData
+# layout: float32_8
-# start: 1332496830.0
+# start: 1332496830.000000
-# end: 1332496830.999
+# end: 1332496830.999000
-1332496830.000000 251774.000000 224241.000000 5688.100098 1915.530029 9329.219727 4183.709961 1212.349976 2641.790039
+1332496830.000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
-1332496830.008333 259567.000000 222698.000000 6207.600098 678.671997 9380.230469 4575.580078 2830.610107 2688.629883
+1332496830.008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
-1332496830.016667 263073.000000 223304.000000 4961.640137 2197.120117 7687.310059 4861.859863 2732.780029 3008.540039
+1332496830.016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03
-1332496830.025000 257614.000000 223323.000000 5003.660156 3525.139893 7165.310059 4685.620117 1715.380005 3440.479980
+1332496830.025000 2.576140e+05 2.233230e+05 5.003660e+03 3.525140e+03 7.165310e+03 4.685620e+03 1.715380e+03 3.440480e+03
-1332496830.033333 255780.000000 221915.000000 6357.310059 2145.290039 8426.969727 3775.350098 1475.390015 3797.239990
+1332496830.033333 2.557800e+05 2.219150e+05 6.357310e+03 2.145290e+03 8.426970e+03 3.775350e+03 1.475390e+03 3.797240e+03
-1332496830.041667 260166.000000 223008.000000 6702.589844 1484.959961 9288.099609 3330.830078 1228.500000 3214.320068
+1332496830.041667 2.601660e+05 2.230080e+05 6.702590e+03 1.484960e+03 9.288100e+03 3.330830e+03 1.228500e+03 3.214320e+03
-1332496830.050000 261231.000000 226426.000000 4980.060059 2982.379883 8499.629883 4267.669922 994.088989 2292.889893
+1332496830.050000 2.612310e+05 2.264260e+05 4.980060e+03 2.982380e+03 8.499630e+03 4.267670e+03 9.940890e+02 2.292890e+03
-1332496830.058333 255117.000000 226642.000000 4584.410156 4656.439941 7860.149902 5317.310059 1473.599976 2111.689941
+1332496830.058333 2.551170e+05 2.266420e+05 4.584410e+03 4.656440e+03 7.860150e+03 5.317310e+03 1.473600e+03 2.111690e+03
-1332496830.066667 253300.000000 223554.000000 6455.089844 3036.649902 8869.750000 4986.310059 2607.360107 2839.590088
+1332496830.066667 2.533000e+05 2.235540e+05 6.455090e+03 3.036650e+03 8.869750e+03 4.986310e+03 2.607360e+03 2.839590e+03
-1332496830.075000 261061.000000 221263.000000 6951.979980 1500.239990 9386.099609 3791.679932 2677.010010 3980.629883
+1332496830.075000 2.610610e+05 2.212630e+05 6.951980e+03 1.500240e+03 9.386100e+03 3.791680e+03 2.677010e+03 3.980630e+03
-1332496830.083333 266503.000000 223198.000000 5189.609863 2594.560059 8571.530273 3175.000000 919.840027 3792.010010
+1332496830.083333 2.665030e+05 2.231980e+05 5.189610e+03 2.594560e+03 8.571530e+03 3.175000e+03 9.198400e+02 3.792010e+03
-1332496830.091667 260692.000000 225184.000000 3782.479980 4642.879883 7662.959961 3917.790039 -251.097000 2907.060059
+1332496830.091667 2.606920e+05 2.251840e+05 3.782480e+03 4.642880e+03 7.662960e+03 3.917790e+03 -2.510970e+02 2.907060e+03
-1332496830.100000 253963.000000 225081.000000 5123.529785 3839.550049 8669.030273 4877.819824 943.723999 2527.449951
+1332496830.100000 2.539630e+05 2.250810e+05 5.123530e+03 3.839550e+03 8.669030e+03 4.877820e+03 9.437240e+02 2.527450e+03
-1332496830.108333 256555.000000 224169.000000 5930.600098 2298.540039 8906.709961 5331.680176 2549.909912 3053.560059
+1332496830.108333 2.565550e+05 2.241690e+05 5.930600e+03 2.298540e+03 8.906710e+03 5.331680e+03 2.549910e+03 3.053560e+03
-1332496830.116667 260889.000000 225010.000000 4681.129883 2971.870117 7900.040039 4874.080078 2322.429932 3649.120117
+1332496830.116667 2.608890e+05 2.250100e+05 4.681130e+03 2.971870e+03 7.900040e+03 4.874080e+03 2.322430e+03 3.649120e+03
-1332496830.125000 257944.000000 224923.000000 3291.139893 4357.089844 7131.589844 4385.560059 1077.050049 3664.040039
+1332496830.125000 2.579440e+05 2.249230e+05 3.291140e+03 4.357090e+03 7.131590e+03 4.385560e+03 1.077050e+03 3.664040e+03
-1332496830.133333 255009.000000 223018.000000 4584.819824 2864.000000 8469.490234 3625.580078 985.557007 3504.229980
+1332496830.133333 2.550090e+05 2.230180e+05 4.584820e+03 2.864000e+03 8.469490e+03 3.625580e+03 9.855570e+02 3.504230e+03
-1332496830.141667 260114.000000 221947.000000 5676.189941 1210.339966 9393.780273 3390.239990 1654.020020 3018.699951
+1332496830.141667 2.601140e+05 2.219470e+05 5.676190e+03 1.210340e+03 9.393780e+03 3.390240e+03 1.654020e+03 3.018700e+03
-1332496830.150000 264277.000000 224438.000000 4446.620117 2176.719971 8142.089844 4584.879883 2327.830078 2615.800049
+1332496830.150000 2.642770e+05 2.244380e+05 4.446620e+03 2.176720e+03 8.142090e+03 4.584880e+03 2.327830e+03 2.615800e+03
-1332496830.158333 259221.000000 226471.000000 2734.439941 4182.759766 6389.549805 5540.520020 1958.880005 2720.120117
+1332496830.158333 2.592210e+05 2.264710e+05 2.734440e+03 4.182760e+03 6.389550e+03 5.540520e+03 1.958880e+03 2.720120e+03
-1332496830.166667 252650.000000 224831.000000 4163.640137 2989.989990 7179.200195 5213.060059 1929.550049 3457.659912
+1332496830.166667 2.526500e+05 2.248310e+05 4.163640e+03 2.989990e+03 7.179200e+03 5.213060e+03 1.929550e+03 3.457660e+03
-1332496830.175000 257083.000000 222048.000000 5759.040039 702.440979 8566.549805 3552.020020 1832.939941 3956.189941
+1332496830.175000 2.570830e+05 2.220480e+05 5.759040e+03 7.024410e+02 8.566550e+03 3.552020e+03 1.832940e+03 3.956190e+03
-1332496830.183333 263130.000000 222967.000000 5141.140137 1166.119995 8666.959961 2720.370117 971.374023 3479.729980
+1332496830.183333 2.631300e+05 2.229670e+05 5.141140e+03 1.166120e+03 8.666960e+03 2.720370e+03 9.713740e+02 3.479730e+03
-1332496830.191667 260236.000000 225265.000000 3425.139893 3339.080078 7853.609863 3674.949951 525.908020 2443.310059
+1332496830.191667 2.602360e+05 2.252650e+05 3.425140e+03 3.339080e+03 7.853610e+03 3.674950e+03 5.259080e+02 2.443310e+03
-1332496830.200000 253503.000000 224527.000000 4398.129883 2927.429932 8110.279785 4842.470215 1513.869995 2467.100098
+1332496830.200000 2.535030e+05 2.245270e+05 4.398130e+03 2.927430e+03 8.110280e+03 4.842470e+03 1.513870e+03 2.467100e+03
-1332496830.208333 256126.000000 222693.000000 6043.529785 656.223999 8797.559570 4832.410156 2832.370117 3426.139893
+1332496830.208333 2.561260e+05 2.226930e+05 6.043530e+03 6.562240e+02 8.797560e+03 4.832410e+03 2.832370e+03 3.426140e+03
-1332496830.216667 261677.000000 223608.000000 5830.459961 1033.910034 8123.939941 3980.689941 1927.959961 4092.719971
+1332496830.216667 2.616770e+05 2.236080e+05 5.830460e+03 1.033910e+03 8.123940e+03 3.980690e+03 1.927960e+03 4.092720e+03
-1332496830.225000 259457.000000 225536.000000 4015.570068 2995.989990 7135.439941 3713.550049 307.220001 3849.429932
+1332496830.225000 2.594570e+05 2.255360e+05 4.015570e+03 2.995990e+03 7.135440e+03 3.713550e+03 3.072200e+02 3.849430e+03
-1332496830.233333 253352.000000 224216.000000 4650.560059 3196.620117 8131.279785 3586.159912 70.832298 3074.179932
+1332496830.233333 2.533520e+05 2.242160e+05 4.650560e+03 3.196620e+03 8.131280e+03 3.586160e+03 7.083230e+01 3.074180e+03
-1332496830.241667 256124.000000 221513.000000 6100.479980 821.979980 9757.540039 3474.510010 1647.520020 2559.860107
+1332496830.241667 2.561240e+05 2.215130e+05 6.100480e+03 8.219800e+02 9.757540e+03 3.474510e+03 1.647520e+03 2.559860e+03
-1332496830.250000 263024.000000 221559.000000 5789.959961 699.416992 9129.740234 4153.080078 2829.250000 2677.270020
+1332496830.250000 2.630240e+05 2.215590e+05 5.789960e+03 6.994170e+02 9.129740e+03 4.153080e+03 2.829250e+03 2.677270e+03
-1332496830.258333 261720.000000 224015.000000 4358.500000 2645.360107 7414.109863 4810.669922 2225.989990 3185.989990
+1332496830.258333 2.617200e+05 2.240150e+05 4.358500e+03 2.645360e+03 7.414110e+03 4.810670e+03 2.225990e+03 3.185990e+03
-1332496830.266667 254756.000000 224240.000000 4857.379883 3229.679932 7539.310059 4769.140137 1507.130005 3668.260010
+1332496830.266667 2.547560e+05 2.242400e+05 4.857380e+03 3.229680e+03 7.539310e+03 4.769140e+03 1.507130e+03 3.668260e+03
-1332496830.275000 256889.000000 222658.000000 6473.419922 1214.109985 9010.759766 3848.729980 1303.839966 3778.500000
+1332496830.275000 2.568890e+05 2.226580e+05 6.473420e+03 1.214110e+03 9.010760e+03 3.848730e+03 1.303840e+03 3.778500e+03
-1332496830.283333 264208.000000 223316.000000 5700.450195 1116.560059 9087.610352 3846.679932 1293.589966 2891.560059
+1332496830.283333 2.642080e+05 2.233160e+05 5.700450e+03 1.116560e+03 9.087610e+03 3.846680e+03 1.293590e+03 2.891560e+03
-1332496830.291667 263310.000000 225719.000000 3936.120117 3252.360107 7552.850098 4897.859863 1156.630005 2037.160034
+1332496830.291667 2.633100e+05 2.257190e+05 3.936120e+03 3.252360e+03 7.552850e+03 4.897860e+03 1.156630e+03 2.037160e+03
-1332496830.300000 255079.000000 225086.000000 4536.450195 3960.110107 7454.589844 5479.069824 1596.359985 2190.800049
+1332496830.300000 2.550790e+05 2.250860e+05 4.536450e+03 3.960110e+03 7.454590e+03 5.479070e+03 1.596360e+03 2.190800e+03
-1332496830.308333 254487.000000 222508.000000 6635.859863 1758.849976 8732.969727 4466.970215 2650.360107 3139.310059
+1332496830.308333 2.544870e+05 2.225080e+05 6.635860e+03 1.758850e+03 8.732970e+03 4.466970e+03 2.650360e+03 3.139310e+03
-1332496830.316667 261241.000000 222432.000000 6702.270020 1085.130005 8989.230469 3112.989990 1933.560059 3828.409912
+1332496830.316667 2.612410e+05 2.224320e+05 6.702270e+03 1.085130e+03 8.989230e+03 3.112990e+03 1.933560e+03 3.828410e+03
-1332496830.325000 262119.000000 225587.000000 4714.950195 2892.360107 8107.819824 2961.310059 239.977997 3273.719971
+1332496830.325000 2.621190e+05 2.255870e+05 4.714950e+03 2.892360e+03 8.107820e+03 2.961310e+03 2.399780e+02 3.273720e+03
-1332496830.333333 254999.000000 226514.000000 4532.089844 4126.899902 8200.129883 3872.590088 56.089001 2370.580078
+1332496830.333333 2.549990e+05 2.265140e+05 4.532090e+03 4.126900e+03 8.200130e+03 3.872590e+03 5.608900e+01 2.370580e+03
-1332496830.341667 254289.000000 224033.000000 6538.810059 2251.439941 9419.429688 4564.450195 2077.810059 2508.169922
+1332496830.341667 2.542890e+05 2.240330e+05 6.538810e+03 2.251440e+03 9.419430e+03 4.564450e+03 2.077810e+03 2.508170e+03
-1332496830.350000 261890.000000 221960.000000 6846.089844 1475.270020 9125.589844 4598.290039 3299.219971 3475.419922
+1332496830.350000 2.618900e+05 2.219600e+05 6.846090e+03 1.475270e+03 9.125590e+03 4.598290e+03 3.299220e+03 3.475420e+03
-1332496830.358333 264502.000000 223085.000000 5066.379883 3270.560059 7933.169922 4173.709961 1908.910034 3867.459961
+1332496830.358333 2.645020e+05 2.230850e+05 5.066380e+03 3.270560e+03 7.933170e+03 4.173710e+03 1.908910e+03 3.867460e+03
-1332496830.366667 257889.000000 223656.000000 4201.660156 4473.640137 7688.339844 4161.580078 687.578979 3653.689941
+1332496830.366667 2.578890e+05 2.236560e+05 4.201660e+03 4.473640e+03 7.688340e+03 4.161580e+03 6.875790e+02 3.653690e+03
-1332496830.375000 254270.000000 223151.000000 5715.140137 2752.139893 9273.320312 3772.949951 896.403992 3256.060059
+1332496830.375000 2.542700e+05 2.231510e+05 5.715140e+03 2.752140e+03 9.273320e+03 3.772950e+03 8.964040e+02 3.256060e+03
-1332496830.383333 258257.000000 224217.000000 6114.310059 1856.859985 9604.320312 4200.490234 1764.380005 2939.219971
+1332496830.383333 2.582570e+05 2.242170e+05 6.114310e+03 1.856860e+03 9.604320e+03 4.200490e+03 1.764380e+03 2.939220e+03
-1332496830.391667 260020.000000 226868.000000 4237.529785 3605.879883 8066.220215 5430.250000 2138.580078 2696.709961
+1332496830.391667 2.600200e+05 2.268680e+05 4.237530e+03 3.605880e+03 8.066220e+03 5.430250e+03 2.138580e+03 2.696710e+03
-1332496830.400000 255083.000000 225924.000000 3350.310059 4853.069824 7045.819824 5925.200195 1893.609985 2897.340088
+1332496830.400000 2.550830e+05 2.259240e+05 3.350310e+03 4.853070e+03 7.045820e+03 5.925200e+03 1.893610e+03 2.897340e+03
-1332496830.408333 254453.000000 222127.000000 5271.330078 2491.500000 8436.679688 5032.080078 2436.050049 3724.590088
+1332496830.408333 2.544530e+05 2.221270e+05 5.271330e+03 2.491500e+03 8.436680e+03 5.032080e+03 2.436050e+03 3.724590e+03
-1332496830.416667 262588.000000 219950.000000 5994.620117 789.273987 9029.650391 3515.739990 1953.569946 4014.520020
+1332496830.416667 2.625880e+05 2.199500e+05 5.994620e+03 7.892740e+02 9.029650e+03 3.515740e+03 1.953570e+03 4.014520e+03
-1332496830.425000 265610.000000 223333.000000 4391.410156 2400.959961 8146.459961 3536.959961 530.231995 3133.919922
+1332496830.425000 2.656100e+05 2.233330e+05 4.391410e+03 2.400960e+03 8.146460e+03 3.536960e+03 5.302320e+02 3.133920e+03
-1332496830.433333 257470.000000 226977.000000 2975.320068 4633.529785 7278.560059 4640.100098 -50.150200 2024.959961
+1332496830.433333 2.574700e+05 2.269770e+05 2.975320e+03 4.633530e+03 7.278560e+03 4.640100e+03 -5.015020e+01 2.024960e+03
-1332496830.441667 250687.000000 226331.000000 4517.859863 3183.800049 8072.600098 5281.660156 1605.140015 2335.139893
+1332496830.441667 2.506870e+05 2.263310e+05 4.517860e+03 3.183800e+03 8.072600e+03 5.281660e+03 1.605140e+03 2.335140e+03
-1332496830.450000 255563.000000 224495.000000 5551.000000 1101.300049 8461.490234 4725.700195 2726.669922 3480.540039
+1332496830.450000 2.555630e+05 2.244950e+05 5.551000e+03 1.101300e+03 8.461490e+03 4.725700e+03 2.726670e+03 3.480540e+03
-1332496830.458333 261335.000000 224645.000000 4764.680176 1557.020020 7833.350098 3524.810059 1577.410034 4038.620117
+1332496830.458333 2.613350e+05 2.246450e+05 4.764680e+03 1.557020e+03 7.833350e+03 3.524810e+03 1.577410e+03 4.038620e+03
-1332496830.466667 260269.000000 224008.000000 3558.030029 2987.610107 7362.439941 3279.229980 562.442017 3786.550049
+1332496830.466667 2.602690e+05 2.240080e+05 3.558030e+03 2.987610e+03 7.362440e+03 3.279230e+03 5.624420e+02 3.786550e+03
-1332496830.475000 257435.000000 221777.000000 4972.600098 2166.879883 8481.440430 3328.719971 1037.130005 3271.370117
+1332496830.475000 2.574350e+05 2.217770e+05 4.972600e+03 2.166880e+03 8.481440e+03 3.328720e+03 1.037130e+03 3.271370e+03
-1332496830.483333 261046.000000 221550.000000 5816.180176 590.216980 9120.929688 3895.399902 2382.669922 2824.169922
+1332496830.483333 2.610460e+05 2.215500e+05 5.816180e+03 5.902170e+02 9.120930e+03 3.895400e+03 2.382670e+03 2.824170e+03
-1332496830.491667 262766.000000 224473.000000 4835.049805 1785.770020 7880.759766 4745.620117 2443.659912 3229.550049
+1332496830.491667 2.627660e+05 2.244730e+05 4.835050e+03 1.785770e+03 7.880760e+03 4.745620e+03 2.443660e+03 3.229550e+03
-1332496830.500000 256509.000000 226413.000000 3758.870117 3461.199951 6743.770020 4928.959961 1536.619995 3546.689941
+1332496830.500000 2.565090e+05 2.264130e+05 3.758870e+03 3.461200e+03 6.743770e+03 4.928960e+03 1.536620e+03 3.546690e+03
-1332496830.508333 250793.000000 224372.000000 5218.490234 2865.260010 7803.959961 4351.089844 1333.819946 3680.489990
+1332496830.508333 2.507930e+05 2.243720e+05 5.218490e+03 2.865260e+03 7.803960e+03 4.351090e+03 1.333820e+03 3.680490e+03
-1332496830.516667 256319.000000 222066.000000 6403.970215 732.344971 9627.759766 3089.300049 1516.780029 3653.689941
+1332496830.516667 2.563190e+05 2.220660e+05 6.403970e+03 7.323450e+02 9.627760e+03 3.089300e+03 1.516780e+03 3.653690e+03
-1332496830.525000 263343.000000 223235.000000 5200.430176 1388.579956 9372.849609 3371.229980 1450.390015 2678.909912
+1332496830.525000 2.633430e+05 2.232350e+05 5.200430e+03 1.388580e+03 9.372850e+03 3.371230e+03 1.450390e+03 2.678910e+03
-1332496830.533333 260903.000000 225110.000000 3722.580078 3246.659912 7876.540039 4716.810059 1498.439941 2116.520020
+1332496830.533333 2.609030e+05 2.251100e+05 3.722580e+03 3.246660e+03 7.876540e+03 4.716810e+03 1.498440e+03 2.116520e+03
-1332496830.541667 254416.000000 223769.000000 4841.649902 2956.399902 8115.919922 5392.359863 2142.810059 2652.320068
+1332496830.541667 2.544160e+05 2.237690e+05 4.841650e+03 2.956400e+03 8.115920e+03 5.392360e+03 2.142810e+03 2.652320e+03
-1332496830.550000 256698.000000 222172.000000 6471.229980 970.395996 8834.980469 4816.839844 2376.629883 3605.860107
+1332496830.550000 2.566980e+05 2.221720e+05 6.471230e+03 9.703960e+02 8.834980e+03 4.816840e+03 2.376630e+03 3.605860e+03
-1332496830.558333 261841.000000 223537.000000 5500.740234 1189.660034 8365.730469 4016.469971 1042.270020 3821.199951
+1332496830.558333 2.618410e+05 2.235370e+05 5.500740e+03 1.189660e+03 8.365730e+03 4.016470e+03 1.042270e+03 3.821200e+03
-1332496830.566667 259503.000000 225840.000000 3827.929932 3088.840088 7676.140137 3978.310059 -357.006989 3016.419922
+1332496830.566667 2.595030e+05 2.258400e+05 3.827930e+03 3.088840e+03 7.676140e+03 3.978310e+03 -3.570070e+02 3.016420e+03
-1332496830.575000 253457.000000 224636.000000 4914.609863 3097.449951 8224.900391 4321.439941 171.373993 2412.360107
+1332496830.575000 2.534570e+05 2.246360e+05 4.914610e+03 3.097450e+03 8.224900e+03 4.321440e+03 1.713740e+02 2.412360e+03
-1332496830.583333 256029.000000 222221.000000 6841.799805 1028.500000 9252.299805 4387.569824 2418.139893 2510.100098
+1332496830.583333 2.560290e+05 2.222210e+05 6.841800e+03 1.028500e+03 9.252300e+03 4.387570e+03 2.418140e+03 2.510100e+03
-1332496830.591667 262840.000000 222550.000000 6210.250000 1410.729980 8538.900391 4152.580078 3009.300049 3219.760010
+1332496830.591667 2.628400e+05 2.225500e+05 6.210250e+03 1.410730e+03 8.538900e+03 4.152580e+03 3.009300e+03 3.219760e+03
-1332496830.600000 261633.000000 225065.000000 4284.529785 3357.209961 7282.169922 3823.590088 1402.839966 3644.669922
+1332496830.600000 2.616330e+05 2.250650e+05 4.284530e+03 3.357210e+03 7.282170e+03 3.823590e+03 1.402840e+03 3.644670e+03
-1332496830.608333 254591.000000 225109.000000 4693.160156 3647.739990 7745.160156 3686.379883 490.161011 3448.860107
+1332496830.608333 2.545910e+05 2.251090e+05 4.693160e+03 3.647740e+03 7.745160e+03 3.686380e+03 4.901610e+02 3.448860e+03
-1332496830.616667 254780.000000 223599.000000 6527.379883 1569.869995 9438.429688 3456.580078 1162.520020 3252.010010
+1332496830.616667 2.547800e+05 2.235990e+05 6.527380e+03 1.569870e+03 9.438430e+03 3.456580e+03 1.162520e+03 3.252010e+03
-1332496830.625000 260639.000000 224107.000000 6531.049805 1633.050049 9283.719727 4174.020020 2089.550049 2775.750000
+1332496830.625000 2.606390e+05 2.241070e+05 6.531050e+03 1.633050e+03 9.283720e+03 4.174020e+03 2.089550e+03 2.775750e+03
-1332496830.633333 261108.000000 225472.000000 4968.259766 3527.850098 7692.870117 5137.100098 2207.389893 2436.659912
+1332496830.633333 2.611080e+05 2.254720e+05 4.968260e+03 3.527850e+03 7.692870e+03 5.137100e+03 2.207390e+03 2.436660e+03
-1332496830.641667 255775.000000 223708.000000 4963.450195 4017.370117 7701.419922 5269.649902 2284.399902 2842.080078
+1332496830.641667 2.557750e+05 2.237080e+05 4.963450e+03 4.017370e+03 7.701420e+03 5.269650e+03 2.284400e+03 2.842080e+03
-1332496830.650000 257398.000000 220947.000000 6767.500000 1645.709961 9107.070312 4000.179932 2548.860107 3624.770020
+1332496830.650000 2.573980e+05 2.209470e+05 6.767500e+03 1.645710e+03 9.107070e+03 4.000180e+03 2.548860e+03 3.624770e+03
-1332496830.658333 264924.000000 221559.000000 6471.459961 1110.329956 9459.650391 3108.169922 1696.969971 3893.439941
+1332496830.658333 2.649240e+05 2.215590e+05 6.471460e+03 1.110330e+03 9.459650e+03 3.108170e+03 1.696970e+03 3.893440e+03
-1332496830.666667 265339.000000 225733.000000 4348.799805 3459.510010 8475.299805 4031.239990 573.346985 2910.270020
+1332496830.666667 2.653390e+05 2.257330e+05 4.348800e+03 3.459510e+03 8.475300e+03 4.031240e+03 5.733470e+02 2.910270e+03
-1332496830.675000 256814.000000 226995.000000 3479.540039 4949.790039 7499.910156 5624.709961 751.656006 2347.709961
+1332496830.675000 2.568140e+05 2.269950e+05 3.479540e+03 4.949790e+03 7.499910e+03 5.624710e+03 7.516560e+02 2.347710e+03
-1332496830.683333 253316.000000 225161.000000 5147.060059 3218.429932 8460.160156 5869.299805 2336.320068 2987.959961
+1332496830.683333 2.533160e+05 2.251610e+05 5.147060e+03 3.218430e+03 8.460160e+03 5.869300e+03 2.336320e+03 2.987960e+03
-1332496830.691667 259360.000000 223101.000000 5549.120117 1869.949951 8740.759766 4668.939941 2457.909912 3758.820068
+1332496830.691667 2.593600e+05 2.231010e+05 5.549120e+03 1.869950e+03 8.740760e+03 4.668940e+03 2.457910e+03 3.758820e+03
-1332496830.700000 262012.000000 224016.000000 4173.609863 3004.129883 8157.040039 3704.729980 987.963989 3652.750000
+1332496830.700000 2.620120e+05 2.240160e+05 4.173610e+03 3.004130e+03 8.157040e+03 3.704730e+03 9.879640e+02 3.652750e+03
-1332496830.708333 257176.000000 224420.000000 3517.300049 4118.750000 7822.240234 3718.229980 37.264900 2953.679932
+1332496830.708333 2.571760e+05 2.244200e+05 3.517300e+03 4.118750e+03 7.822240e+03 3.718230e+03 3.726490e+01 2.953680e+03
-1332496830.716667 255146.000000 223322.000000 4923.979980 2330.679932 9095.910156 3792.399902 1013.070007 2711.239990
+1332496830.716667 2.551460e+05 2.233220e+05 4.923980e+03 2.330680e+03 9.095910e+03 3.792400e+03 1.013070e+03 2.711240e+03
-1332496830.725000 260524.000000 223651.000000 5413.629883 1146.209961 8817.169922 4419.649902 2446.649902 2832.050049
+1332496830.725000 2.605240e+05 2.236510e+05 5.413630e+03 1.146210e+03 8.817170e+03 4.419650e+03 2.446650e+03 2.832050e+03
-1332496830.733333 262098.000000 225752.000000 4262.979980 2270.969971 7135.479980 5067.120117 2294.679932 3376.620117
+1332496830.733333 2.620980e+05 2.257520e+05 4.262980e+03 2.270970e+03 7.135480e+03 5.067120e+03 2.294680e+03 3.376620e+03
-1332496830.741667 256889.000000 225379.000000 3606.459961 3568.189941 6552.649902 4970.270020 1516.380005 3662.570068
+1332496830.741667 2.568890e+05 2.253790e+05 3.606460e+03 3.568190e+03 6.552650e+03 4.970270e+03 1.516380e+03 3.662570e+03
-1332496830.750000 253948.000000 222631.000000 5511.700195 2066.300049 7952.660156 4019.909912 1513.140015 3752.629883
+1332496830.750000 2.539480e+05 2.226310e+05 5.511700e+03 2.066300e+03 7.952660e+03 4.019910e+03 1.513140e+03 3.752630e+03
-1332496830.758333 259799.000000 222067.000000 5873.500000 608.583984 9253.780273 2870.739990 1348.239990 3344.199951
+1332496830.758333 2.597990e+05 2.220670e+05 5.873500e+03 6.085840e+02 9.253780e+03 2.870740e+03 1.348240e+03 3.344200e+03
-1332496830.766667 262547.000000 224901.000000 4346.080078 1928.099976 8590.969727 3455.459961 904.390991 2379.270020
+1332496830.766667 2.625470e+05 2.249010e+05 4.346080e+03 1.928100e+03 8.590970e+03 3.455460e+03 9.043910e+02 2.379270e+03
-1332496830.775000 256137.000000 226761.000000 3423.560059 3379.080078 7471.149902 4894.169922 1153.540039 2031.410034
+1332496830.775000 2.561370e+05 2.267610e+05 3.423560e+03 3.379080e+03 7.471150e+03 4.894170e+03 1.153540e+03 2.031410e+03
-1332496830.783333 250326.000000 225013.000000 5519.979980 2423.969971 7991.759766 5117.950195 2098.790039 3099.239990
+1332496830.783333 2.503260e+05 2.250130e+05 5.519980e+03 2.423970e+03 7.991760e+03 5.117950e+03 2.098790e+03 3.099240e+03
-1332496830.791667 255454.000000 222992.000000 6547.950195 496.496002 8751.339844 3900.560059 2132.290039 4076.810059
+1332496830.791667 2.554540e+05 2.229920e+05 6.547950e+03 4.964960e+02 8.751340e+03 3.900560e+03 2.132290e+03 4.076810e+03
-1332496830.800000 261286.000000 223489.000000 5152.850098 1501.510010 8425.610352 2888.030029 776.114014 3786.360107
+1332496830.800000 2.612860e+05 2.234890e+05 5.152850e+03 1.501510e+03 8.425610e+03 2.888030e+03 7.761140e+02 3.786360e+03
-1332496830.808333 258969.000000 224069.000000 3832.610107 3001.979980 7979.259766 3182.310059 52.716000 2874.800049
+1332496830.808333 2.589690e+05 2.240690e+05 3.832610e+03 3.001980e+03 7.979260e+03 3.182310e+03 5.271600e+01 2.874800e+03
-1332496830.816667 254946.000000 222035.000000 5317.879883 2139.800049 9103.139648 3955.610107 1235.170044 2394.149902
+1332496830.816667 2.549460e+05 2.220350e+05 5.317880e+03 2.139800e+03 9.103140e+03 3.955610e+03 1.235170e+03 2.394150e+03
-1332496830.825000 258676.000000 221205.000000 6594.910156 505.343994 9423.360352 4562.470215 2913.739990 2892.350098
+1332496830.825000 2.586760e+05 2.212050e+05 6.594910e+03 5.053440e+02 9.423360e+03 4.562470e+03 2.913740e+03 2.892350e+03
-1332496830.833333 262125.000000 223566.000000 5116.750000 1773.599976 8082.200195 4776.370117 2386.389893 3659.729980
+1332496830.833333 2.621250e+05 2.235660e+05 5.116750e+03 1.773600e+03 8.082200e+03 4.776370e+03 2.386390e+03 3.659730e+03
-1332496830.841667 257835.000000 225918.000000 3714.300049 3477.080078 7205.370117 4554.609863 711.539001 3878.419922
+1332496830.841667 2.578350e+05 2.259180e+05 3.714300e+03 3.477080e+03 7.205370e+03 4.554610e+03 7.115390e+02 3.878420e+03
-1332496830.850000 253660.000000 224371.000000 5022.450195 2592.429932 8277.200195 4119.370117 486.507996 3666.739990
+1332496830.850000 2.536600e+05 2.243710e+05 5.022450e+03 2.592430e+03 8.277200e+03 4.119370e+03 4.865080e+02 3.666740e+03
-1332496830.858333 259503.000000 222061.000000 6589.950195 659.935974 9596.919922 3598.100098 1702.489990 3036.600098
+1332496830.858333 2.595030e+05 2.220610e+05 6.589950e+03 6.599360e+02 9.596920e+03 3.598100e+03 1.702490e+03 3.036600e+03
-1332496830.866667 265495.000000 222843.000000 5541.850098 1728.430054 8459.959961 4492.000000 2231.969971 2430.620117
+1332496830.866667 2.654950e+05 2.228430e+05 5.541850e+03 1.728430e+03 8.459960e+03 4.492000e+03 2.231970e+03 2.430620e+03
-1332496830.875000 260929.000000 224996.000000 4000.949951 3745.989990 6983.790039 5430.859863 1855.260010 2533.379883
+1332496830.875000 2.609290e+05 2.249960e+05 4.000950e+03 3.745990e+03 6.983790e+03 5.430860e+03 1.855260e+03 2.533380e+03
-1332496830.883333 252716.000000 224335.000000 5086.560059 3401.149902 7597.970215 5196.120117 1755.719971 3079.760010
+1332496830.883333 2.527160e+05 2.243350e+05 5.086560e+03 3.401150e+03 7.597970e+03 5.196120e+03 1.755720e+03 3.079760e+03
-1332496830.891667 254110.000000 223111.000000 6822.189941 1229.079956 9164.339844 3761.229980 1679.390015 3584.879883
+1332496830.891667 2.541100e+05 2.231110e+05 6.822190e+03 1.229080e+03 9.164340e+03 3.761230e+03 1.679390e+03 3.584880e+03
-1332496830.900000 259969.000000 224693.000000 6183.950195 1538.500000 9222.080078 3139.169922 949.901978 3180.800049
+1332496830.900000 2.599690e+05 2.246930e+05 6.183950e+03 1.538500e+03 9.222080e+03 3.139170e+03 9.499020e+02 3.180800e+03
-1332496830.908333 259078.000000 226913.000000 4388.890137 3694.820068 8195.019531 3933.000000 426.079987 2388.449951
+1332496830.908333 2.590780e+05 2.269130e+05 4.388890e+03 3.694820e+03 8.195020e+03 3.933000e+03 4.260800e+02 2.388450e+03
-1332496830.916667 254563.000000 224760.000000 5168.439941 4020.939941 8450.269531 4758.910156 1458.900024 2286.429932
+1332496830.916667 2.545630e+05 2.247600e+05 5.168440e+03 4.020940e+03 8.450270e+03 4.758910e+03 1.458900e+03 2.286430e+03
-1332496830.925000 258059.000000 221217.000000 6883.459961 1649.530029 9232.780273 4457.649902 3057.820068 3031.949951
+1332496830.925000 2.580590e+05 2.212170e+05 6.883460e+03 1.649530e+03 9.232780e+03 4.457650e+03 3.057820e+03 3.031950e+03
-1332496830.933333 264667.000000 221177.000000 6218.509766 1645.729980 8657.179688 3663.500000 2528.280029 3978.340088
+1332496830.933333 2.646670e+05 2.211770e+05 6.218510e+03 1.645730e+03 8.657180e+03 3.663500e+03 2.528280e+03 3.978340e+03
-1332496830.941667 262925.000000 224382.000000 4627.500000 3635.929932 7892.799805 3431.320068 604.508972 3901.370117
+1332496830.941667 2.629250e+05 2.243820e+05 4.627500e+03 3.635930e+03 7.892800e+03 3.431320e+03 6.045090e+02 3.901370e+03
-1332496830.950000 254708.000000 225448.000000 4408.250000 4461.040039 8197.169922 3953.750000 -44.534599 3154.870117
+1332496830.950000 2.547080e+05 2.254480e+05 4.408250e+03 4.461040e+03 8.197170e+03 3.953750e+03 -4.453460e+01 3.154870e+03
-1332496830.958333 253702.000000 224635.000000 5825.770020 2577.050049 9590.049805 4569.250000 1460.270020 2785.169922
+1332496830.958333 2.537020e+05 2.246350e+05 5.825770e+03 2.577050e+03 9.590050e+03 4.569250e+03 1.460270e+03 2.785170e+03
-1332496830.966667 260206.000000 224140.000000 5387.979980 1951.160034 8789.509766 5131.660156 2706.379883 2972.479980
+1332496830.966667 2.602060e+05 2.241400e+05 5.387980e+03 1.951160e+03 8.789510e+03 5.131660e+03 2.706380e+03 2.972480e+03
-1332496830.975000 261240.000000 224737.000000 3860.810059 3418.310059 7414.529785 5284.520020 2271.379883 3183.149902
+1332496830.975000 2.612400e+05 2.247370e+05 3.860810e+03 3.418310e+03 7.414530e+03 5.284520e+03 2.271380e+03 3.183150e+03
-1332496830.983333 256140.000000 223252.000000 3850.010010 3957.139893 7262.649902 4964.640137 1499.510010 3453.129883
+1332496830.983333 2.561400e+05 2.232520e+05 3.850010e+03 3.957140e+03 7.262650e+03 4.964640e+03 1.499510e+03 3.453130e+03
-1332496830.991667 256116.000000 221349.000000 5594.479980 2054.399902 8835.129883 3662.010010 1485.510010 3613.010010
+1332496830.991667 2.561160e+05 2.213490e+05 5.594480e+03 2.054400e+03 8.835130e+03 3.662010e+03 1.485510e+03 3.613010e+03

tests/data/prep-20120323T1000

@@ -1,3 +1,4 @@
+# comments are cool?
 2.66568e+05  2.24029e+05  5.16140e+03  2.52517e+03  8.35084e+03  3.72470e+03  1.35534e+03  2.03900e+03  
 2.57914e+05  2.27183e+05  4.30368e+03  4.13080e+03  7.25535e+03  4.89047e+03  1.63859e+03  1.93496e+03  
 2.51717e+05  2.26047e+05  5.99445e+03  3.49363e+03  8.07250e+03  5.08267e+03  2.26917e+03  2.86231e+03  

tests/test_bulkdata.py

@@ -35,8 +35,6 @@ class TestBulkData(object):
             data.create("/foo", "uint16_8")
         with assert_raises(ValueError):
             data.create("foo/bar", "uint16_8")
-        with assert_raises(ValueError):
-            data.create("/foo/bar", "uint8_8")
         data.create("/foo/bar", "uint16_8")
         data.create(u"/foo/baz/quux", "float64_16")
         with assert_raises(ValueError):
@@ -70,6 +68,13 @@ class TestBulkData(object):
         for s in misc_slices:
             eq_(node[s], raw[s])
 
+        # Extract misc slices while appending, to make sure the
+        # data isn't being added in the middle of the file
+        for s in [2, slice(1,5), 2, slice(1,5)]:
+            node.append([[0,0,0,0,0,0,0,0,0]])
+            raw.append([0,0,0,0,0,0,0,0,0])
+            eq_(node[s], raw[s])
+
         # Get some coverage of remove; remove is more fully tested
         # in cmdline
         with assert_raises(IndexError):

tests/test_client.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
-import nilmdb
+import nilmdb.server
+import nilmdb.client
+
 from nilmdb.utils.printf import *
 from nilmdb.utils import timestamper
 from nilmdb.client import ClientError, ServerError
@@ -20,11 +22,12 @@ import unittest
 import warnings
 import resource
 import time
+import re
 
 from testutil.helpers import *
 
 testdb = "tests/client-testdb"
-testurl = "http://localhost:12380/"
+testurl = "http://localhost:32180/"
 
 def setup_module():
     global test_server, test_db
@@ -32,11 +35,11 @@ def setup_module():
     recursive_unlink(testdb)
 
     # Start web app on a custom port
-    test_db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(testdb, sync = False)
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(testdb)
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
+    test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+                                       port = 32180, stoppable = False,
-                                fast_shutdown = True,
+                                       fast_shutdown = True,
-                                force_traceback = False)
+                                       force_traceback = True)
     test_server.start(blocking = False)
 
 def teardown_module():
@@ -49,30 +52,24 @@ class TestClient(object):
 
     def test_client_01_basic(self):
         # Test a fake host
-        client = nilmdb.Client(url = "http://localhost:1/")
+        client = nilmdb.client.Client(url = "http://localhost:1/")
-        with assert_raises(nilmdb.client.ServerError):
-            client.version()
-        client.close()
-
-        # Trigger same error with a PUT request
-        client = nilmdb.Client(url = "http://localhost:1/")
         with assert_raises(nilmdb.client.ServerError):
             client.version()
         client.close()
 
         # Then a fake URL on a real host
-        client = nilmdb.Client(url = "http://localhost:12380/fake/")
+        client = nilmdb.client.Client(url = "http://localhost:32180/fake/")
         with assert_raises(nilmdb.client.ClientError):
             client.version()
         client.close()
 
         # Now a real URL with no http:// prefix
-        client = nilmdb.Client(url = "localhost:12380")
+        client = nilmdb.client.Client(url = "localhost:32180")
         version = client.version()
         client.close()
 
         # Now use the real URL
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
         version = client.version()
         eq_(distutils.version.LooseVersion(version),
             distutils.version.LooseVersion(test_server.version))
@@ -84,16 +81,16 @@ class TestClient(object):
 
     def test_client_02_createlist(self):
         # Basic stream tests, like those in test_nilmdb:test_stream
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
 
         # Database starts empty
         eq_(client.stream_list(), [])
 
         # Bad path
         with assert_raises(ClientError):
-            client.stream_create("foo/bar/baz", "PrepData")
+            client.stream_create("foo/bar/baz", "float32_8")
         with assert_raises(ClientError):
-            client.stream_create("/foo", "PrepData")
+            client.stream_create("/foo", "float32_8")
         # Bad layout type
         with assert_raises(ClientError):
             client.stream_create("/newton/prep", "NoSuchLayout")
@@ -105,23 +102,23 @@ class TestClient(object):
         for x in range(10):
             with assert_raises(ClientError):
                 client.http.post("/stream/list")
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
 
         # Create three streams
-        client.stream_create("/newton/prep", "PrepData")
+        client.stream_create("/newton/prep", "float32_8")
-        client.stream_create("/newton/raw", "RawData")
+        client.stream_create("/newton/raw", "uint16_6")
-        client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
+        client.stream_create("/newton/zzz/rawnotch", "uint16_9")
 
         # Verify we got 3 streams
-        eq_(client.stream_list(), [ ["/newton/prep", "PrepData"],
+        eq_(client.stream_list(), [ ["/newton/prep", "float32_8"],
-                                    ["/newton/raw", "RawData"],
+                                    ["/newton/raw", "uint16_6"],
-                                    ["/newton/zzz/rawnotch", "RawNotchedData"]
+                                    ["/newton/zzz/rawnotch", "uint16_9"]
                                     ])
         # Match just one type or one path
-        eq_(client.stream_list(layout="RawData"),
+        eq_(client.stream_list(layout="uint16_6"),
-            [ ["/newton/raw", "RawData"] ])
+            [ ["/newton/raw", "uint16_6"] ])
         eq_(client.stream_list(path="/newton/raw"),
-            [ ["/newton/raw", "RawData"] ])
+            [ ["/newton/raw", "uint16_6"] ])
 
         # Try messing with resource limits to trigger errors and get
         # more coverage.  Here, make it so we can only create files 1
@@ -130,13 +127,13 @@ class TestClient(object):
         limit = resource.getrlimit(resource.RLIMIT_FSIZE)
         resource.setrlimit(resource.RLIMIT_FSIZE, (1, limit[1]))
         with assert_raises(ServerError) as e:
-            client.stream_create("/newton/hello", "RawData")
+            client.stream_create("/newton/hello", "uint16_6")
         resource.setrlimit(resource.RLIMIT_FSIZE, limit)
 
         client.close()
 
     def test_client_03_metadata(self):
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
 
         # Set / get metadata
         eq_(client.stream_get_metadata("/newton/prep"), {})
@@ -167,10 +164,24 @@ class TestClient(object):
             client.stream_set_metadata("/newton/prep", [1,2,3])
         with assert_raises(ClientError):
             client.stream_update_metadata("/newton/prep", [1,2,3])
+
+        # test wrong types (dict of non-strings)
+        # numbers are OK; they'll get converted to strings
+        client.stream_set_metadata("/newton/prep", { "hello": 1234 })
+        # anything else is not
+        with assert_raises(ClientError):
+            client.stream_set_metadata("/newton/prep", { "world": { 1: 2 } })
+        with assert_raises(ClientError):
+            client.stream_set_metadata("/newton/prep", { "world": [ 1, 2 ] })
+
         client.close()
 
     def test_client_04_insert(self):
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
+
+        # Limit _max_data to 1 MB, since our test file is 1.5 MB
+        old_max_data = nilmdb.client.client.StreamInserter._max_data
+        nilmdb.client.client.StreamInserter._max_data = 1 * 1024 * 1024
 
         datetime_tz.localtz_set("America/New_York")
 
@@ -191,7 +202,8 @@ class TestClient(object):
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
         in_("400 Bad Request", str(e.exception))
-        in_("timestamp is not monotonically increasing", str(e.exception))
+        in2_("timestamp is not monotonically increasing",
+             "start must precede end", str(e.exception))
 
         # Now try empty data (no server request made)
         empty = cStringIO.StringIO("")
@@ -233,7 +245,7 @@ class TestClient(object):
             result = client.stream_insert("/newton/prep", data,
                                           start + 5, start + 120)
         in_("400 Bad Request", str(e.exception))
-        in_("Data timestamp 1332511200.0 < start time 1332511205.0",
+        in_("Data timestamp 1332511200.000000 < start time 1332511205.000000",
             str(e.exception))
 
         # Specify start/end (ends too early)
@@ -244,8 +256,9 @@ class TestClient(object):
         in_("400 Bad Request", str(e.exception))
         # Client chunks the input, so the exact timestamp here might change
         # if the chunk positions change.
-        in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
+        assert(re.search("Data timestamp 13325[0-9]+\.[0-9]+ "
-            str(e.exception))
+                         ">= end time 1332511201.000000", str(e.exception))
+               is not None)
 
         # Now do the real load
         data = timestamper.TimestamperRate(testfile, start, 120)
@@ -264,11 +277,12 @@ class TestClient(object):
         in_("400 Bad Request", str(e.exception))
         in_("verlap", str(e.exception))
 
+        nilmdb.client.client.StreamInserter._max_data = old_max_data
         client.close()
 
     def test_client_05_extractremove(self):
         # Misc tests for extract and remove.  Most of them are in test_cmdline.
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
 
         for x in client.stream_extract("/newton/prep", 999123, 999124):
             raise AssertionError("shouldn't be any data for this request")
@@ -284,7 +298,7 @@ class TestClient(object):
     def test_client_06_generators(self):
         # A lot of the client functionality is already tested by test_cmdline,
         # but this gets a bit more coverage that cmdline misses.
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
 
         # Trigger a client error in generator
         start = datetime_tz.datetime_tz.smartparse("20120323T2000")
@@ -320,7 +334,7 @@ class TestClient(object):
         # Pokes around in client.http internals a bit to look at the
         # response headers.
 
-        client = nilmdb.Client(url = testurl)
+        client = nilmdb.client.Client(url = testurl)
         http = client.http
 
         # Use a warning rather than returning a test failure for the
@@ -353,52 +367,48 @@ class TestClient(object):
             raise AssertionError("/stream/extract is not text/plain:\n" +
                                  headers())
 
-        # Make sure Access-Control-Allow-Origin gets set
-        if "access-control-allow-origin: " not in headers():
-            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
-                                 "header in /stream/extract response:\n" +
-                                 headers())
-
         client.close()
 
     def test_client_08_unicode(self):
-        # Basic Unicode tests
+        # Try both with and without posting JSON
-        client = nilmdb.Client(url = testurl)
+        for post_json in (False, True):
+            # Basic Unicode tests
+            client = nilmdb.client.Client(url = testurl, post_json = post_json)
 
-        # Delete streams that exist
+            # Delete streams that exist
-        for stream in client.stream_list():
+            for stream in client.stream_list():
-            client.stream_destroy(stream[0])
+                client.stream_destroy(stream[0])
 
-        # Database is empty
+            # Database is empty
-        eq_(client.stream_list(), [])
+            eq_(client.stream_list(), [])
 
-        # Create Unicode stream, match it
+            # Create Unicode stream, match it
-        raw = [ u"/düsseldorf/raw", u"uint16_6" ]
+            raw = [ u"/düsseldorf/raw", u"uint16_6" ]
-        prep = [ u"/düsseldorf/prep", u"uint16_6" ]
+            prep = [ u"/düsseldorf/prep", u"uint16_6" ]
-        client.stream_create(*raw)
+            client.stream_create(*raw)
-        eq_(client.stream_list(), [raw])
+            eq_(client.stream_list(), [raw])
-        eq_(client.stream_list(layout=raw[1]), [raw])
+            eq_(client.stream_list(layout=raw[1]), [raw])
-        eq_(client.stream_list(path=raw[0]), [raw])
+            eq_(client.stream_list(path=raw[0]), [raw])
-        client.stream_create(*prep)
+            client.stream_create(*prep)
-        eq_(client.stream_list(), [prep, raw])
+            eq_(client.stream_list(), [prep, raw])
 
-        # Set / get metadata with Unicode keys and values
+            # Set / get metadata with Unicode keys and values
-        eq_(client.stream_get_metadata(raw[0]), {})
+            eq_(client.stream_get_metadata(raw[0]), {})
-        eq_(client.stream_get_metadata(prep[0]), {})
+            eq_(client.stream_get_metadata(prep[0]), {})
-        meta1 = { u"alpha": u"α",
+            meta1 = { u"alpha": u"α",
-                  u"β": u"beta" }
+                      u"β": u"beta" }
-        meta2 = { u"alpha": u"α" }
+            meta2 = { u"alpha": u"α" }
-        meta3 = { u"β": u"beta" }
+            meta3 = { u"β": u"beta" }
-        client.stream_set_metadata(prep[0], meta1)
+            client.stream_set_metadata(prep[0], meta1)
-        client.stream_update_metadata(prep[0], {})
+            client.stream_update_metadata(prep[0], {})
-        client.stream_update_metadata(raw[0], meta2)
+            client.stream_update_metadata(raw[0], meta2)
-        client.stream_update_metadata(raw[0], meta3)
+            client.stream_update_metadata(raw[0], meta3)
-        eq_(client.stream_get_metadata(prep[0]), meta1)
+            eq_(client.stream_get_metadata(prep[0]), meta1)
-        eq_(client.stream_get_metadata(raw[0]), meta1)
+            eq_(client.stream_get_metadata(raw[0]), meta1)
-        eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
+            eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
-        eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
+            eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
 
-        client.close()
+            client.close()
 
     def test_client_09_closing(self):
         # Make sure we actually close sockets correctly.  New
@@ -412,68 +422,86 @@ class TestClient(object):
                                          "probably not closing properly.")
                 if test == 1:
                     # explicit close
-                    client = nilmdb.Client(url = testurl)
+                    client = nilmdb.client.Client(url = testurl)
                     with assert_raises(ClientError) as e:
                         client.stream_remove("/newton/prep", 123, 120)
                     client.close() # remove this to see the failure
                 elif test == 2:
                     # use the context manager
-                    with nilmdb.Client(url = testurl) as c:
+                    with nilmdb.client.Client(url = testurl) as c:
                         with assert_raises(ClientError) as e:
                             c.stream_remove("/newton/prep", 123, 120)
 
     def test_client_10_context(self):
         # Test using the client's stream insertion context manager to
         # insert data.
-        client = nilmdb.Client(testurl)
+        client = nilmdb.client.Client(testurl)
 
         client.stream_create("/context/test", "uint16_1")
         with client.stream_insert_context("/context/test") as ctx:
             # override _max_data to trigger frequent server updates
             ctx._max_data = 15
 
-            with assert_raises(ValueError):
+            ctx.insert("100 1\n")
-                ctx.insert_line("100 1")
 
-            ctx.insert_line("100 1\n")
+            ctx.insert("101 ")
-            ctx.insert_iter([ "101 1\n",
+            ctx.insert("1\n102 1")
-                              "102 1\n",
+            ctx.insert("")
-                              "103 1\n" ])
+            ctx.insert("\n103 1\n")
-            ctx.insert_line("104 1\n")
+
-            ctx.insert_line("105 1\n")
+            ctx.insert("104 1\n")
+            ctx.insert("# hello\n")
+            ctx.insert("   # hello\n")
+            ctx.insert("  105 1\n")
             ctx.finalize()
 
-            ctx.insert_line("106 1\n")
+            ctx.insert("106 1\n")
             ctx.update_end(106.5)
             ctx.finalize()
             ctx.update_start(106.8)
-            ctx.insert_line("107 1\n")
+            ctx.insert("107 1\n")
-            ctx.insert_line("108 1\n")
+            ctx.insert("108 1\n")
-            ctx.insert_line("109 1\n")
+            ctx.insert("109 1\n")
-            ctx.insert_line("110 1\n")
+            ctx.insert("110 1\n")
-            ctx.insert_line("111 1\n")
+            ctx.insert("111 1\n")
             ctx.update_end(113)
-            ctx.insert_line("112 1\n")
+            ctx.insert("112 1\n")
             ctx.update_end(114)
-            ctx.insert_line("113 1\n")
+            ctx.insert("113 1\n")
             ctx.update_end(115)
-            ctx.insert_line("114 1\n")
+            ctx.insert("114 1" +
+                       " # this is super long" * 100 +
+                       "\n")
             ctx.finalize()
+            ctx.insert("# this is super long" * 100)
 
         with assert_raises(ClientError):
             with client.stream_insert_context("/context/test", 100, 200) as ctx:
-                ctx.insert_line("115 1\n")
+                ctx.insert("115 1\n")
 
         with assert_raises(ClientError):
             with client.stream_insert_context("/context/test", 200, 300) as ctx:
-                ctx.insert_line("115 1\n")
+                ctx.insert("115 1\n")
+
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test") as ctx:
+                ctx.insert("bogus data\n")
 
         with client.stream_insert_context("/context/test", 200, 300) as ctx:
             # make sure our override wasn't permanent
             ne_(ctx._max_data, 15)
-            ctx.insert_line("225 1\n")
+            ctx.insert("225 1\n")
             ctx.finalize()
 
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test", 300, 400) as ctx:
+                ctx.insert("301 1\n")
+                ctx.insert("302 2\n")
+                ctx.insert("303 3\n")
+                ctx.insert("304 4\n")
+                ctx.insert("304 4\n") # non-monotonic after a few lines
+                ctx.finalize()
+
         eq_(list(client.stream_intervals("/context/test")),
             [ [ 100, 105.000001 ],
               [ 106, 106.5 ],
@@ -487,7 +515,7 @@ class TestClient(object):
         # Empty intervals are ok!  If recording detection events
         # by inserting rows into the database, we want to be able to
         # have an interval where no events occurred.  Test them here.
-        client = nilmdb.Client(testurl)
+        client = nilmdb.client.Client(testurl)
         client.stream_create("/empty/test", "uint16_1")
 
         def info():
@@ -502,9 +530,9 @@ class TestClient(object):
         # Insert a region with just a few points
         with client.stream_insert_context("/empty/test") as ctx:
             ctx.update_start(100)
-            ctx.insert_line("140 1\n")
+            ctx.insert("140 1\n")
-            ctx.insert_line("150 1\n")
+            ctx.insert("150 1\n")
-            ctx.insert_line("160 1\n")
+            ctx.insert("160 1\n")
             ctx.update_end(200)
             ctx.finalize()
 
@@ -517,7 +545,7 @@ class TestClient(object):
 
         # Try also creating a completely empty interval from scratch,
         # in a few different ways.
-        client.stream_insert_block("/empty/test", "", 300, 350)
+        client.stream_insert("/empty/test", "", 300, 350)
         client.stream_insert("/empty/test", [], 400, 450)
         with client.stream_insert_context("/empty/test", 500, 550):
             pass
@@ -538,10 +566,10 @@ class TestClient(object):
             ctx.finalize() # inserts [1000, 1050]
             ctx.finalize() # nothing
             ctx.finalize() # nothing
-            ctx.insert_line("1100 1\n")
+            ctx.insert("1100 1\n")
             ctx.finalize() # inserts [1100, 1100.000001]
             ctx.update_start(1199)
-            ctx.insert_line("1200 1\n")
+            ctx.insert("1200 1\n")
             ctx.update_end(1250)
             ctx.finalize() # inserts [1199, 1250]
             ctx.update_start(1299)
@@ -549,8 +577,15 @@ class TestClient(object):
             ctx.update_end(1350)
             ctx.finalize() # nothing
             ctx.update_start(1400)
+            ctx.insert("# nothing!\n")
             ctx.update_end(1450)
             ctx.finalize()
+            ctx.update_start(1500)
+            ctx.insert("# nothing!")
+            ctx.update_end(1550)
+            ctx.finalize()
+            ctx.insert("# nothing!\n" * 10)
+            ctx.finalize()
             # implicit last finalize inserts [1400, 1450]
 
         # Check everything
@@ -563,6 +598,7 @@ class TestClient(object):
                      (1, [1100, 1100.000001]),
                      (1, [1199, 1250]),
                      (0, [1400, 1450]),
+                     (0, [1500, 1550]),
                      ])
 
         # Clean up
@@ -573,11 +609,11 @@ class TestClient(object):
         # Check that connections are persistent when they should be.
         # This is pretty hard to test; we have to poke deep into
         # the Requests library.
-        with nilmdb.Client(url = testurl) as c:
+        with nilmdb.client.Client(url = testurl) as c:
             def connections():
                 try:
                     poolmanager = c.http._last_response.connection.poolmanager
-                    pool = poolmanager.pools[('http','localhost',12380)]
+                    pool = poolmanager.pools[('http','localhost',32180)]
                     return (pool.num_connections, pool.num_requests)
                 except:
                     raise SkipTest("can't get connection info")

tests/test_cmdline.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
-import nilmdb
+import nilmdb.server
+
 from nilmdb.utils.printf import *
 import nilmdb.cmdline
 from nilmdb.utils import datetime_tz
@@ -11,12 +12,7 @@ from nose.tools import assert_raises
 import itertools
 import os
 import re
-import shutil
 import sys
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
 import StringIO
 import shlex
 
@@ -27,14 +23,14 @@ testdb = "tests/cmdline-testdb"
 def server_start(max_results = None, bulkdata_args = {}):
     global test_server, test_db
     # Start web app on a custom port
-    test_db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(
-        testdb, sync = False,
+        testdb,
         max_results = max_results,
         bulkdata_args = bulkdata_args)
-    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
+    test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
-                                port = 12380, stoppable = False,
+                                       port = 32180, stoppable = False,
-                                fast_shutdown = True,
+                                       fast_shutdown = True,
-                                force_traceback = False)
+                                       force_traceback = False)
     test_server.start(blocking = False)
 
 def server_stop():
@@ -64,6 +60,7 @@ class TestCmdline(object):
         passing the given input.  Returns a tuple with the output and
         exit code"""
         # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
+        os.environ['NILMDB_URL'] = "http://localhost:32180/"
         class stdio_wrapper:
             def __init__(self, stdin, stdout, stderr):
                 self.io = (stdin, stdout, stderr)
@@ -129,8 +126,17 @@ class TestCmdline(object):
         with open(file) as f:
             contents = f.read()
             if contents != self.captured:
-                print contents[1:1000] + "\n"
+                print "--- reference file (first 1000 bytes):\n"
-                print self.captured[1:1000] + "\n"
+                print contents[0:1000] + "\n"
+                print "--- captured data (first 1000 bytes):\n"
+                print self.captured[0:1000] + "\n"
+                zipped = itertools.izip_longest(contents, self.captured)
+                for (n, (a, b)) in enumerate(zipped):
+                    if a != b:
+                        print "--- first difference is at offset", n
+                        print "--- reference:", repr(a)
+                        print "---  captured:", repr(b)
+                        break
                 raise AssertionError("captured data doesn't match " + file)
 
     def matchfilecount(self, file):
@@ -174,7 +180,7 @@ class TestCmdline(object):
         self.fail("-u localhost:1 info")
         self.contain("error connecting to server")
 
-        self.ok("-u localhost:12380 info")
+        self.ok("-u localhost:32180 info")
         self.ok("info")
 
         # Duplicated arguments should fail, but this isn't implemented
@@ -192,6 +198,20 @@ class TestCmdline(object):
             self.fail("extract --start 2000-01-01 --start 2001-01-02")
             self.contain("duplicated argument")
 
+        # Verify that "help command" and "command --help" are identical
+        # for all commands.
+        self.fail("")
+        m = re.search(r"{(.*)}", self.captured)
+        for command in [""] + m.group(1).split(','):
+            self.ok(command + " --help")
+            cap1 = self.captured
+            self.ok("help " + command)
+            cap2 = self.captured
+            self.ok("help " + command + " asdf --url --zxcv -")
+            cap3 = self.captured
+            eq_(cap1, cap2)
+            eq_(cap2, cap3)
+
     def test_02_parsetime(self):
         os.environ['TZ'] = "America/New_York"
         test = datetime_tz.datetime_tz.now()
@@ -210,7 +230,7 @@ class TestCmdline(object):
 
     def test_03_info(self):
         self.ok("info")
-        self.contain("Server URL: http://localhost:12380/")
+        self.contain("Server URL: http://localhost:32180/")
         self.contain("Client version: " + nilmdb.__version__)
         self.contain("Server version: " + test_server.version)
         self.contain("Server database path")
@@ -225,10 +245,10 @@ class TestCmdline(object):
         self.match("")
 
         # Bad paths
-        self.fail("create foo/bar/baz PrepData")
+        self.fail("create foo/bar/baz float32_8")
         self.contain("paths must start with /")
 
-        self.fail("create /foo PrepData")
+        self.fail("create /foo float32_8")
         self.contain("invalid path")
 
         # Bad layout type
@@ -240,53 +260,53 @@ class TestCmdline(object):
         self.contain("no such layout")
 
         # Create a few streams
-        self.ok("create /newton/zzz/rawnotch RawNotchedData")
+        self.ok("create /newton/zzz/rawnotch uint16_9")
-        self.ok("create /newton/prep PrepData")
+        self.ok("create /newton/prep float32_8")
-        self.ok("create /newton/raw RawData")
+        self.ok("create /newton/raw uint16_6")
 
         # Should not be able to create a stream with another stream as
         # its parent
-        self.fail("create /newton/prep/blah PrepData")
+        self.fail("create /newton/prep/blah float32_8")
         self.contain("path is subdir of existing node")
 
         # Should not be able to create a stream at a location that
         # has other nodes as children
-        self.fail("create /newton/zzz PrepData")
+        self.fail("create /newton/zzz float32_8")
         self.contain("subdirs of this path already exist")
 
         # Verify we got those 3 streams and they're returned in
         # alphabetical order.
         self.ok("list")
-        self.match("/newton/prep PrepData\n"
+        self.match("/newton/prep float32_8\n"
-                   "/newton/raw RawData\n"
+                   "/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         # Match just one type or one path.  Also check
         # that --path is optional
         self.ok("list --path /newton/raw")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         self.ok("list /newton/raw")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         self.fail("list -p /newton/raw /newton/raw")
         self.contain("too many paths")
 
-        self.ok("list --layout RawData")
+        self.ok("list --layout uint16_6")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         # Wildcard matches
-        self.ok("list --layout Raw*")
+        self.ok("list --layout uint16*")
-        self.match("/newton/raw RawData\n"
+        self.match("/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list --path *zzz* --layout Raw*")
+        self.ok("list --path *zzz* --layout uint16*")
-        self.match("/newton/zzz/rawnotch RawNotchedData\n")
+        self.match("/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list *zzz* --layout Raw*")
+        self.ok("list *zzz* --layout uint16*")
-        self.match("/newton/zzz/rawnotch RawNotchedData\n")
+        self.match("/newton/zzz/rawnotch uint16_9\n")
 
-        self.ok("list --path *zzz* --layout Prep*")
+        self.ok("list --path *zzz* --layout float32*")
         self.match("")
 
         # reversed range
@@ -353,43 +373,49 @@ class TestCmdline(object):
     def test_06_insert(self):
         self.ok("insert --help")
 
-        self.fail("insert /foo/bar baz qwer")
+        self.fail("insert -s 2000 -e 2001 /foo/bar baz")
         self.contain("error getting stream info")
 
-        self.fail("insert /newton/prep baz qwer")
+        self.fail("insert -s 2000 -e 2001 /newton/prep baz")
         self.match("error opening input file baz\n")
 
-        self.fail("insert /newton/prep")
+        self.fail("insert /newton/prep --timestamp -f -r 120")
-        self.contain("error extracting time")
+        self.contain("error extracting start time")
 
-        self.fail("insert --start 19801205 /newton/prep 1 2 3 4")
+        self.fail("insert /newton/prep --timestamp -r 120")
-        self.contain("--start can only be used with one input file")
+        self.contain("need --start or --filename")
 
         self.fail("insert /newton/prep "
                   "tests/data/prep-20120323T1000")
 
-        # insert pre-timestamped data, from stdin
-        os.environ['TZ'] = "UTC"
-        with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
-
         # insert pre-timestamped data, with bad times (non-monotonic)
         os.environ['TZ'] = "UTC"
         with open("tests/data/prep-20120323T1004-badtimes") as input:
-            self.fail("insert --none /newton/prep", input)
+            self.fail("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                      input)
             self.contain("error parsing input data")
             self.contain("line 7:")
             self.contain("timestamp is not monotonically increasing")
 
+        # insert pre-timestamped data, from stdin
+        os.environ['TZ'] = "UTC"
+        with open("tests/data/prep-20120323T1004-timestamped") as input:
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
+
         # insert data with normal timestamper from filename
         os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
+        self.ok("insert --timestamp -f --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1000 "
+                "tests/data/prep-20120323T1000")
+        self.fail("insert -t --filename /newton/prep "
+                "tests/data/prep-20120323T1002")
+        self.contain("rate is needed")
+        self.ok("insert -t --filename --rate 120 /newton/prep "
                 "tests/data/prep-20120323T1002")
 
         # overlap
         os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/prep "
+        self.fail("insert --timestamp -f --rate 120 /newton/prep "
                   "tests/data/prep-20120323T1004")
         self.contain("overlap")
 
@@ -401,24 +427,24 @@ class TestCmdline(object):
 
         # still an overlap if we specify a different start
         os.environ['TZ'] = "America/New_York"
-        self.fail("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep"
+        self.fail("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep"
                   " tests/data/prep-20120323T1004")
         self.contain("overlap")
 
         # wrong format
         os.environ['TZ'] = "UTC"
-        self.fail("insert --rate 120 /newton/raw "
+        self.fail("insert -t -r 120 -f /newton/raw "
                   "tests/data/prep-20120323T1004")
         self.contain("error parsing input data")
 
         # empty data does nothing
-        self.ok("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep "
+        self.ok("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep "
                 "/dev/null")
 
         # bad start time
-        self.fail("insert --rate 120 --start 'whatever' /newton/prep /dev/null")
+        self.fail("insert -t -r 120 --start 'whatever' /newton/prep /dev/null")
 
-    def test_07_detail(self):
+    def test_07_detail_extended(self):
         # Just count the number of lines, it's probably fine
         self.ok("list --detail")
         lines_(self.captured, 8)
@@ -456,12 +482,29 @@ class TestCmdline(object):
         self.ok("list --detail --path *prep --timestamp-raw "
                 "--start='23 Mar 2012 10:05:15.50'")
         lines_(self.captured, 2)
-        self.contain("[ 1332497115.5 -> 1332497159.991668 ]")
+        self.contain("[ 1332497115.500000 -> 1332497160.000000 ]")
 
-        self.ok("list --detail --path *prep -T "
+        # bad time
-                "--start='23 Mar 2012 10:05:15.612'")
+        self.fail("list --detail --path *prep -T --start='9332497115.612'")
+        # good time
+        self.ok("list --detail --path *prep -T --start='1332497115.612'")
         lines_(self.captured, 2)
-        self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
+        self.contain("[ 1332497115.612000 -> 1332497160.000000 ]")
+
+        # Check --ext output
+        self.ok("list --ext")
+        lines_(self.captured, 9)
+
+        self.ok("list -E -T")
+        c = self.contain
+        c("\n  interval extents: 1332496800.000000 -> 1332497160.000000\n")
+        c("\n        total data: 43200 rows, 359.983336 seconds\n")
+        c("\n  interval extents: (no data)\n")
+        c("\n        total data: 0 rows, 0.000000 seconds\n")
+
+        # Misc
+        self.fail("list --ext --start='23 Mar 2012 10:05:15.50'")
+        self.contain("--start and --end only make sense with --detail")
 
     def test_08_extract(self):
         # nonexistent stream
@@ -572,13 +615,13 @@ class TestCmdline(object):
 
         # Make sure we have the data we expect
         self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:04:00.000000 +0000"
-                   " -> Fri, 23 Mar 2012 10:05:59.991668 +0000 ]\n")
+                   " -> Fri, 23 Mar 2012 10:06:00.000000 +0000 ]\n")
 
         # Remove various chunks of prep data and make sure
         # they're gone.
@@ -607,7 +650,7 @@ class TestCmdline(object):
 
         # See the missing chunks in list output
         self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
@@ -621,14 +664,15 @@ class TestCmdline(object):
         self.ok("remove /newton/prep --start 2000-01-01 --end 2020-01-01")
         self.match("")  # no count requested this time
         self.ok("list --detail /newton/prep")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  (no intervals)\n")
 
         # Reinsert some data, to verify that no overlaps with deleted
         # data are reported
         os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
+        self.ok("insert --timestamp -f --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1000 "
+                "tests/data/prep-20120323T1000")
+        self.ok("insert -t --filename --rate 120 /newton/prep "
                 "tests/data/prep-20120323T1002")
 
     def test_11_destroy(self):
@@ -646,9 +690,9 @@ class TestCmdline(object):
 
         # From previous tests, we have:
         self.ok("list")
-        self.match("/newton/prep PrepData\n"
+        self.match("/newton/prep float32_8\n"
-                   "/newton/raw RawData\n"
+                   "/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         # Notice how they're not empty
         self.ok("list --detail")
@@ -657,15 +701,15 @@ class TestCmdline(object):
         # Delete some
         self.ok("destroy /newton/prep")
         self.ok("list")
-        self.match("/newton/raw RawData\n"
+        self.match("/newton/raw uint16_6\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
+                   "/newton/zzz/rawnotch uint16_9\n")
 
         self.ok("destroy /newton/zzz/rawnotch")
         self.ok("list")
-        self.match("/newton/raw RawData\n")
+        self.match("/newton/raw uint16_6\n")
 
         self.ok("destroy /newton/raw")
-        self.ok("create /newton/raw RawData")
+        self.ok("create /newton/raw uint16_6")
         self.ok("destroy /newton/raw")
         self.ok("list")
         self.match("")
@@ -675,7 +719,7 @@ class TestCmdline(object):
                     "/newton/raw", "/newton/asdf/qwer" ]
         for path in rebuild:
             # Create the path
-            self.ok("create " + path + " PrepData")
+            self.ok("create " + path + " float32_8")
             self.ok("list")
             self.contain(path)
             # Make sure it was created empty
@@ -713,7 +757,8 @@ class TestCmdline(object):
         self.ok("create /newton/prep float32_8")
         os.environ['TZ'] = "UTC"
         with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
 
         # Extract it
         self.ok("extract /newton/prep --start '2000-01-01' " +
@@ -745,7 +790,8 @@ class TestCmdline(object):
         self.ok("create /newton/prep float32_8")
         os.environ['TZ'] = "UTC"
         with open("tests/data/prep-20120323T1004-timestamped") as input:
-            self.ok("insert --none /newton/prep", input)
+            self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
+                    input)
         nfiles = 0
         for (dirpath, dirnames, filenames) in os.walk(testdb):
             nfiles += len(filenames)
@@ -761,10 +807,11 @@ class TestCmdline(object):
                                        "files_per_dir" : 3 })
 
         # Insert data.  Just for fun, insert out of order
-        self.ok("create /newton/prep PrepData")
+        self.ok("create /newton/prep float32_8")
         os.environ['TZ'] = "UTC"
-        self.ok("insert --rate 120 /newton/prep "
+        self.ok("insert -t --filename --rate 120 /newton/prep "
-                "tests/data/prep-20120323T1002 "
+                "tests/data/prep-20120323T1002")
+        self.ok("insert -t --filename --rate 120 /newton/prep "
                 "tests/data/prep-20120323T1000")
 
         # Should take up about 2.8 MB here (including directory entries)
@@ -772,7 +819,7 @@ class TestCmdline(object):
 
         # Make sure we have the data we expect
         self.ok("list --detail")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
@@ -808,7 +855,7 @@ class TestCmdline(object):
 
         # See the missing chunks in list output
         self.ok("list --detail")
-        self.match("/newton/prep PrepData\n" +
+        self.match("/newton/prep float32_8\n" +
                    "  [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
                    " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
                    "  [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
@@ -829,7 +876,7 @@ class TestCmdline(object):
         # With the specific file_size above, this will cause the last
         # file in the bulk data storage to be exactly file_size large,
         # so removing the data should also remove that last file.
-        self.ok("insert --rate 120 /newton/prep " +
+        self.ok("insert --timestamp -f --rate 120 /newton/prep " +
                 "tests/data/prep-20120323T1002-first19lines")
         self.ok("remove /newton/prep " +
                 "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
@@ -840,7 +887,8 @@ class TestCmdline(object):
 
         # Re-add the full 10:02 data file.  This tests adding new data once
         # we removed data near the end.
-        self.ok("insert --rate 120 /newton/prep tests/data/prep-20120323T1002")
+        self.ok("insert -t -f -r 120 /newton/prep "
+                "tests/data/prep-20120323T1002")
 
         # See if we can extract it all
         self.ok("extract /newton/prep --start 2000-01-01 --end 2020-01-01")

tests/test_interval.py

@@ -89,14 +89,14 @@ class TestInterval:
 
         # big integers and floats
         x = Interval(5000111222, 6000111222)
-        eq_(str(x), "[5000111222.0 -> 6000111222.0)")
+        eq_(str(x), "[5000111222.000000 -> 6000111222.000000)")
         x = Interval(123.45, 234.56)
-        eq_(str(x), "[123.45 -> 234.56)")
+        eq_(str(x), "[123.450000 -> 234.560000)")
 
         # misc
         i = Interval(d1, d2)
         eq_(repr(i), repr(eval(repr(i))))
-        eq_(str(i), "[1332561600.0 -> 1332648000.0)")
+        eq_(str(i), "[1332561600.000000 -> 1332648000.000000)")
 
     def test_interval_intersect(self):
         # Test Interval intersections
@@ -192,7 +192,8 @@ class TestInterval:
 
         # misc
         eq_(repr(iset), repr(eval(repr(iset))))
-        eq_(str(iset), "[[100.0 -> 200.0), [200.0 -> 300.0)]")
+        eq_(str(iset),
+            "[[100.000000 -> 200.000000), [200.000000 -> 300.000000)]")
 
     def test_intervalset_geniset(self):
         # Test basic iset construction

tests/test_layout.py

@@ -9,14 +9,7 @@ from nose.tools import assert_raises
 import distutils.version
 import itertools
 import os
-import shutil
 import sys
-import cherrypy
-import threading
-import urllib2
-from urllib2 import urlopen, HTTPError
-import Queue
-import cStringIO
 import random
 import unittest
 
@@ -28,7 +21,7 @@ class TestLayouts(object):
     # Some nilmdb.layout tests.  Not complete, just fills in missing
     # coverage.
     def test_layouts(self):
-        x = nilmdb.server.layout.get_named("PrepData")
+        x = nilmdb.server.layout.get_named("float32_8")
         y = nilmdb.server.layout.get_named("float32_8")
         eq_(x.count, y.count)
         eq_(x.datatype, y.datatype)
@@ -37,7 +30,7 @@ class TestLayouts(object):
         eq_(x.datatype, y.datatype)
 
     def test_parsing(self):
-        self.real_t_parsing("PrepData", "RawData", "RawNotchedData")
+        self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
         self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
     def real_t_parsing(self, name_prep, name_raw, name_rawnotch):
         # invalid layouts
@@ -72,7 +65,7 @@ class TestLayouts(object):
         eq_(parser.data, [[1234567890.0,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8],
                           [1234567890.1,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]])
 
-        # try RawData too, with clamping
+        # try uint16_6 too, with clamping
         parser = Parser(name_raw)
         data = ( "1234567890.000000 1 2 3 4 5 6\n" +
                  "1234567890.100000 1 2 3 4 5 6\n" )
@@ -106,7 +99,7 @@ class TestLayouts(object):
                  "1234567890.100001 1 2 3 4 5 6\n" )
         parser.parse(data)
 
-        # RawData with values out of bounds
+        # uint16_6 with values out of bounds
         parser = Parser(name_raw)
         data = ( "1234567890.000000 1 2 3 4 500000 6\n" +
                  "1234567890.100000 1 2 3 4 5 6\n" )
@@ -122,7 +115,7 @@ class TestLayouts(object):
         assert(parser.max_timestamp is None)
 
     def test_formatting(self):
-        self.real_t_formatting("PrepData", "RawData", "RawNotchedData")
+        self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
         self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
     def real_t_formatting(self, name_prep, name_raw, name_rawnotch):
         # invalid layout
@@ -151,12 +144,14 @@ class TestLayouts(object):
                  [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ] ]
         text = formatter.format(data)
         eq_(text,
-            "1234567890.000000 1.100000 2.200000 3.300000 4.400000 " +
+            "1234567890.000000 1.100000e+00 2.200000e+00 3.300000e+00 "
-            "5.500000 6.600000 7.700000 8.800000\n" +
+            "4.400000e+00 5.500000e+00 6.600000e+00 7.700000e+00 "
-            "1234567890.100000 1.100000 2.200000 3.300000 4.400000 " +
+            "8.800000e+00\n" +
-            "5.500000 6.600000 7.700000 8.800000\n")
+            "1234567890.100000 1.100000e+00 2.200000e+00 3.300000e+00 "
+            "4.400000e+00 5.500000e+00 6.600000e+00 7.700000e+00 "
+            "8.800000e+00\n")
 
-        # try RawData too
+        # try uint16_6 too
         formatter = Formatter(name_raw)
         data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6 ],
                  [ 1234567890.100000, 1, 2, 3, 4, 5, 6 ] ]
@@ -181,7 +176,7 @@ class TestLayouts(object):
         eq_(text, "")
 
     def test_roundtrip(self):
-        self.real_t_roundtrip("PrepData", "RawData", "RawNotchedData")
+        self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
         self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
     def real_t_roundtrip(self, name_prep, name_raw, name_rawnotch):
         # Verify that textual data passed into the Parser, and then
@@ -208,7 +203,7 @@ class TestLayouts(object):
                 eq_(parser1.data, parser2.data)
 
         def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
+            return [ sprintf("%.6e", random.uniform(-1000,1000))
                      for x in range(8) ]
         do_roundtrip(name_prep, datagen)
 
@@ -246,7 +241,7 @@ class TestLayoutSpeed:
                 parser = Parser(layout)
                 formatter = Formatter(layout)
                 parser.parse(data)
-                data = formatter.format(parser.data)
+                formatter.format(parser.data)
             elapsed = time.time() - start
             printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
                    layout,
@@ -256,7 +251,7 @@ class TestLayoutSpeed:
 
         print ""
         def datagen():
-            return [ sprintf("%f", random.uniform(-1000,1000))
+            return [ sprintf("%.6e", random.uniform(-1000,1000))
                      for x in range(10) ]
         do_speedtest("float32_10", datagen)
 
@@ -264,3 +259,8 @@ class TestLayoutSpeed:
             return [ sprintf("%d", random.randint(0,65535))
                      for x in range(10) ]
         do_speedtest("uint16_10", datagen)
+
+        def datagen():
+            return [ sprintf("%d", random.randint(0,65535))
+                     for x in range(6) ]
+        do_speedtest("uint16_6", datagen)

tests/test_mustclose.py

@@ -34,6 +34,10 @@ class Bar:
     def __del__(self):
         fprintf(err, "Deleting\n")
 
+    @classmethod
+    def baz(self):
+        fprintf(err, "Baz\n")
+
     def close(self):
         fprintf(err, "Closing\n")
 

tests/test_nilmdb.py

@@ -1,4 +1,4 @@
-import nilmdb
+import nilmdb.server
 
 from nose.tools import *
 from nose.tools import assert_raises
@@ -6,15 +6,13 @@ import distutils.version
 import simplejson as json
 import itertools
 import os
-import shutil
 import sys
-import cherrypy
 import threading
 import urllib2
 from urllib2 import urlopen, HTTPError
-import Queue
 import cStringIO
 import time
+import requests
 
 from nilmdb.utils import serializer_proxy
 
@@ -31,11 +29,11 @@ class Test00Nilmdb(object):  # named 00 so it runs first
         recursive_unlink(testdb)
 
         with assert_raises(IOError):
-            nilmdb.NilmDB("/nonexistant-db/foo")
+            nilmdb.server.NilmDB("/nonexistant-db/foo")
 
-        db = nilmdb.NilmDB(testdb)
+        db = nilmdb.server.NilmDB(testdb)
         db.close()
-        db = nilmdb.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         db.close()
 
         # test timer, just to get coverage
@@ -48,29 +46,29 @@ class Test00Nilmdb(object):  # named 00 so it runs first
         in_("test: ", capture.getvalue())
 
     def test_stream(self):
-        db = nilmdb.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         eq_(db.stream_list(), [])
 
         # Bad path
         with assert_raises(ValueError):
-            db.stream_create("foo/bar/baz", "PrepData")
+            db.stream_create("foo/bar/baz", "float32_8")
         with assert_raises(ValueError):
-            db.stream_create("/foo", "PrepData")
+            db.stream_create("/foo", "float32_8")
         # Bad layout type
         with assert_raises(ValueError):
             db.stream_create("/newton/prep", "NoSuchLayout")
-        db.stream_create("/newton/prep", "PrepData")
+        db.stream_create("/newton/prep", "float32_8")
-        db.stream_create("/newton/raw", "RawData")
+        db.stream_create("/newton/raw", "uint16_6")
-        db.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
+        db.stream_create("/newton/zzz/rawnotch", "uint16_9")
 
         # Verify we got 3 streams
-        eq_(db.stream_list(), [ ["/newton/prep", "PrepData"],
+        eq_(db.stream_list(), [ ["/newton/prep", "float32_8"],
-                                ["/newton/raw", "RawData"],
+                                ["/newton/raw", "uint16_6"],
-                                ["/newton/zzz/rawnotch", "RawNotchedData"]
+                                ["/newton/zzz/rawnotch", "uint16_9"]
                                 ])
         # Match just one type or one path
-        eq_(db.stream_list(layout="RawData"), [ ["/newton/raw", "RawData"] ])
+        eq_(db.stream_list(layout="uint16_6"), [ ["/newton/raw", "uint16_6"] ])
-        eq_(db.stream_list(path="/newton/raw"), [ ["/newton/raw", "RawData"] ])
+        eq_(db.stream_list(path="/newton/raw"), [ ["/newton/raw", "uint16_6"] ])
 
         # Verify that columns were made right (pytables specific)
         if "h5file" in db.data.__dict__:
@@ -106,7 +104,7 @@ class Test00Nilmdb(object):  # named 00 so it runs first
 
 class TestBlockingServer(object):
     def setUp(self):
-        self.db = serializer_proxy(nilmdb.NilmDB)(testdb, sync=False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
 
     def tearDown(self):
         self.db.close()
@@ -115,11 +113,11 @@ class TestBlockingServer(object):
         # Server should fail if the database doesn't have a "_thread_safe"
         # property.
         with assert_raises(KeyError):
-            nilmdb.Server(object())
+            nilmdb.server.Server(object())
 
         # Start web app on a custom port
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
+        self.server = nilmdb.server.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = True)
+                                           port = 32180, stoppable = True)
 
         # Run it
         event = threading.Event()
@@ -131,13 +129,13 @@ class TestBlockingServer(object):
             raise AssertionError("server didn't start in 10 seconds")
 
         # Send request to exit.
-        req = urlopen("http://127.0.0.1:12380/exit/", timeout = 1)
+        req = urlopen("http://127.0.0.1:32180/exit/", timeout = 1)
 
         # Wait for it
         thread.join()
 
 def geturl(path):
-    req = urlopen("http://127.0.0.1:12380" + path, timeout = 10)
+    req = urlopen("http://127.0.0.1:32180" + path, timeout = 10)
     return req.read()
 
 def getjson(path):
@@ -147,9 +145,9 @@ class TestServer(object):
 
     def setUp(self):
         # Start web app on a custom port
-        self.db = serializer_proxy(nilmdb.NilmDB)(testdb, sync=False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
-        self.server = nilmdb.Server(self.db, host = "127.0.0.1",
+        self.server = nilmdb.server.Server(self.db, host = "127.0.0.1",
-                                    port = 12380, stoppable = False)
+                                           port = 32180, stoppable = False)
         self.server.start(blocking = False)
 
     def tearDown(self):
@@ -173,13 +171,13 @@ class TestServer(object):
         streams = getjson("/stream/list")
 
         eq_(streams, [
-            ['/newton/prep', 'PrepData'],
+            ['/newton/prep', 'float32_8'],
-            ['/newton/raw', 'RawData'],
+            ['/newton/raw', 'uint16_6'],
-            ['/newton/zzz/rawnotch', 'RawNotchedData'],
+            ['/newton/zzz/rawnotch', 'uint16_9'],
             ])
 
-        streams = getjson("/stream/list?layout=RawData")
+        streams = getjson("/stream/list?layout=uint16_6")
-        eq_(streams, [['/newton/raw', 'RawData']])
+        eq_(streams, [['/newton/raw', 'uint16_6']])
 
         streams = getjson("/stream/list?layout=NoSuchLayout")
         eq_(streams, [])
@@ -208,3 +206,51 @@ class TestServer(object):
         data = getjson("/stream/get_metadata?path=/newton/prep"
                        "&key=foo")
         eq_(data, {'foo': None})
+
+    def test_cors_headers(self):
+        # Test that CORS headers are being set correctly
+
+        # Normal GET should send simple response
+        url = "http://127.0.0.1:32180/stream/list"
+        r = requests.get(url, headers = { "Origin": "http://google.com/" })
+        eq_(r.status_code, 200)
+        if "access-control-allow-origin" not in r.headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-origin"], "http://google.com/")
+
+        # OPTIONS without CORS preflight headers should result in 405
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            })
+        eq_(r.status_code, 405)
+
+        # OPTIONS with preflight headers should give preflight response
+        r = requests.options(url, headers = {
+            "Origin": "http://google.com/",
+            "Access-Control-Request-Method": "POST",
+            "Access-Control-Request-Headers": "X-Custom",
+            })
+        eq_(r.status_code, 200)
+        if "access-control-allow-origin" not in r.headers:
+            raise AssertionError("No Access-Control-Allow-Origin (CORS) "
+                                 "header in response:\n", r.headers)
+        eq_(r.headers["access-control-allow-methods"], "GET, HEAD")
+        eq_(r.headers["access-control-allow-headers"], "X-Custom")
+
+    def test_post_bodies(self):
+        # Test JSON post bodies
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '{"hello": 1}')
+        eq_(r.status_code, 404) # wrong parameters
+
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '["hello"]')
+        eq_(r.status_code, 415) # not a dict
+
+        r = requests.post("http://127.0.0.1:32180/stream/set_metadata",
+                          headers = { "Content-Type": "application/json" },
+                          data = '[hello]')
+        eq_(r.status_code, 400) # badly formatted JSON

tests/test_timestamper.py

@@ -83,10 +83,3 @@ class TestTimestamper(object):
         for line in ts:
             raise AssertionError
         ts.close()
-
-        # Test the null timestamper
-        input = cStringIO.StringIO(join(lines_out))  # note: lines_out
-        ts = timestamper.TimestamperNull(input)
-        foo = ts.readlines()
-        eq_(foo, join(lines_out))
-        eq_(str(ts), "TimestamperNull(...)")

tests/testutil/helpers.py

@@ -20,6 +20,11 @@ def in_(a, b):
     if a not in b:
         raise AssertionError("%s not in %s" % (myrepr(a), myrepr(b)))
 
+def in2_(a1, a2, b):
+    if a1 not in b and a2 not in b:
+        raise AssertionError("(%s or %s) not in %s" % (myrepr(a1), myrepr(a2),
+                                                       myrepr(b)))
+
 def ne_(a, b):
     if not a != b:
         raise AssertionError("unexpected %s == %s" % (myrepr(a), myrepr(b)))