Browse Source

Start reworking/fixing insert timestamps

Jim Paris 9 years ago
2 changed files with 14 additions and 34 deletions
  1. +7
  2. +7

+ 7
- 19
nilmdb/ View File

@@ -383,30 +383,18 @@ class NilmDB(object):
con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
con.execute("DELETE FROM streams WHERE id=?", (stream_id,))

def stream_insert(self, path, parser, old_timestamp = None):
def stream_insert(self, path, start, end, data):
"""Insert new data into the database.
path: Path at which to add the data
parser: nilmdb.layout.Parser instance full of data to insert
start: Starting timestamp
end: Ending timestamp
data: Rows of data, to be passed to PyTable's table.append
method. E.g.
if (not parser.min_timestamp or not parser.max_timestamp or
not len(
raise StreamError("no data provided")

# If we were provided with an old timestamp, the expectation
# is that the client has a contiguous block of time it is sending,
# but it's doing it over multiple calls to stream_insert.
# old_timestamp is the max_timestamp of the previous insert.
# To make things continuous, use that as our starting timestamp
# instead of what the parser found.
if old_timestamp:
min_timestamp = old_timestamp
min_timestamp = parser.min_timestamp

# First check for basic overlap using timestamp info given.
stream_id = self._stream_id(path)
iset = self._get_intervals(stream_id)
interval = Interval(min_timestamp, parser.max_timestamp)
interval = Interval(start, end)
if iset.intersects(interval):
raise OverlapError("new data overlaps existing data at range: "
+ str(iset & interval))
@@ -414,7 +402,7 @@ class NilmDB(object):
# Insert the data into pytables
table = self.h5file.getNode(path)
row_start = table.nrows
row_end = table.nrows

+ 7
- 15
nilmdb/ View File

@@ -156,17 +156,10 @@ class Stream(NilmApp):
def insert(self, path, old_timestamp = None):
def insert(self, path, start, end):
Insert new data into the database. Provide textual data
(matching the path's layout) as a HTTP PUT.

old_timestamp is used when making multiple, split-up insertions
for a larger contiguous block of data. The first insert
will return the maximum timestamp that it saw, and the second
insert should provide this timestamp as an argument. This is
used to extend the previous database interval rather than
start a new one.

# Important that we always read the input before throwing any
@@ -195,16 +188,15 @@ class Stream(NilmApp):

# Now do the nilmdb insert, passing it the parser full of data.
if old_timestamp:
old_timestamp = float(old_timestamp)
result = self.db.stream_insert(path, parser, old_timestamp)
result = self.db.stream_insert(path,
except nilmdb.nilmdb.NilmDBError as e:
raise cherrypy.HTTPError("400 Bad Request", e.message)

# Return the maximum timestamp that we saw. The client will
# return this back to us as the old_timestamp parameter, if
# it has more data to send.
return ("ok", parser.max_timestamp)
# Done
return "ok"

# /stream/intervals?path=/newton/prep
# /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0