git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10655 ddd99763-3ecb-0310-9145-efcb8ce7c51ftags/bxinterval-last
@@ -74,11 +74,11 @@ Converting from ASCII to PyTables: | |||||
- Server converts this ACSII data to a list of values | - Server converts this ACSII data to a list of values | ||||
- Maybe: | - Maybe: | ||||
# threaded side creates this object | |||||
parser = nilmdb.layout.Parser("layout_name") | |||||
# threaded side creates this object | |||||
parser = nilmdb.layout.Parser("layout_name") | |||||
# threaded side parses and fills it with data | # threaded side parses and fills it with data | ||||
parser.parse(textdata) | parser.parse(textdata) | ||||
# serialized side pulls out rows | # serialized side pulls out rows | ||||
for n in xrange(parser.nrows): | for n in xrange(parser.nrows): | ||||
parser.fill_row(rowinstance) | |||||
parser.fill_row(rowinstance, n) | |||||
table.append() | table.append() |
@@ -1,8 +1,8 @@ | |||||
"""Main NilmDB import""" | """Main NilmDB import""" | ||||
from .nilmdb import NilmDB, StreamError, OverlapError | |||||
from .nilmdb import NilmDB | |||||
from .server import Server | from .server import Server | ||||
from .client import Client, ClientError, ServerError | |||||
from .client import Client | |||||
import layout | import layout | ||||
import serializer | import serializer | ||||
@@ -72,6 +72,7 @@ class Parser(object): | |||||
self.layout = named[layout] | self.layout = named[layout] | ||||
self.data = [] | self.data = [] | ||||
self.nrows = 0 | |||||
def parse(self, textdata): | def parse(self, textdata): | ||||
"""Parse the data, provided as lines of text, using the current | """Parse the data, provided as lines of text, using the current | ||||
@@ -80,16 +81,20 @@ class Parser(object): | |||||
# This currently takes about 0.1 seconds for 1 megabyte of prep data, | # This currently takes about 0.1 seconds for 1 megabyte of prep data, | ||||
# 85 klines/sec. Could clearly be optimized a lot... | # 85 klines/sec. Could clearly be optimized a lot... | ||||
indata = cStringIO.StringIO(textdata) | indata = cStringIO.StringIO(textdata) | ||||
n = 0 | |||||
self.nrows = 0 | |||||
# Assume any parsing error is a real error. | |||||
# In the future we might want to skip completely empty lines, | |||||
# or partial lines right before EOF? | |||||
try: | try: | ||||
for line in indata: | for line in indata: | ||||
n += 1 | |||||
self.nrows += 1 | |||||
fields = line.partition('#')[0].split() | fields = line.partition('#')[0].split() | ||||
self.data.append(self.layout.parse(fields)) | self.data.append(self.layout.parse(fields)) | ||||
except (ValueError, TypeError, IndexError) as e: | except (ValueError, TypeError, IndexError) as e: | ||||
raise TypeError("line " + n + ": " + e.message) | |||||
raise TypeError("line " + self.nrows + ": " + e.message) | |||||
def fillrow(self, row): | |||||
"""Fill a PyTables row object that matches the layout with the | |||||
stored data""" | |||||
def fillrow(self, tablerow, rownum): | |||||
"""Fill a PyTables row object with the parsed data. | |||||
The row must match the parser's layout""" | |||||
for (n, (name, type)) in enumerate(self.layout.fields): | |||||
tablerow[name] = self.data[rownum][n] |
@@ -260,12 +260,13 @@ class NilmDB(object): | |||||
data.update(newdata) | data.update(newdata) | ||||
self.stream_set_metadata(path, data) | self.stream_set_metadata(path, data) | ||||
def stream_insert(self, path, data): | |||||
def stream_insert(self, path, parser): | |||||
"""Insert new data into the database. | """Insert new data into the database. | ||||
path: Path of data | |||||
data: Data to insert | |||||
path: Path at which to add the data | |||||
parser: nilmdb.layout.Parser instance full of data to insert | |||||
""" | """ | ||||
pass | |||||
# First check for basic overlap using timestamp info from the parser. | |||||
raise NilmDBError("go away") | |||||
@@ -91,7 +91,7 @@ class Stream(NilmApp): | |||||
matching the given keys.""" | matching the given keys.""" | ||||
try: | try: | ||||
data = self.db.stream_get_metadata(path) | data = self.db.stream_get_metadata(path) | ||||
except nilmdb.StreamError as e: | |||||
except nilmdb.nilmdb.StreamError as e: | |||||
raise cherrypy.HTTPError("404 Not Found", e.message) | raise cherrypy.HTTPError("404 Not Found", e.message) | ||||
if key is None: # If no keys specified, return them all | if key is None: # If no keys specified, return them all | ||||
key = data.keys() | key = data.keys() | ||||
@@ -169,18 +169,14 @@ class Stream(NilmApp): | |||||
raise cherrypy.HTTPError("400 Bad Request", | raise cherrypy.HTTPError("400 Bad Request", | ||||
"Error parsing input data: " + | "Error parsing input data: " + | ||||
e.message) | e.message) | ||||
return "ok" | |||||
# XXX TODO | |||||
print time.time() | |||||
# Do the magic | |||||
# Now do the nilmdb insert, passing it the parser full of data. | |||||
try: | try: | ||||
result = self.db.stream_insert(path, body) | |||||
except nilmdb.OverlapError as e: | |||||
result = self.db.stream_insert(path, parser) | |||||
except nilmdb.nilmdb.NilmDBError as e: | |||||
raise cherrypy.HTTPError("400 Bad Request", e.message) | raise cherrypy.HTTPError("400 Bad Request", e.message) | ||||
print time.time() | |||||
return "ok" | return "ok" | ||||
raise cherrypy.HTTPError("501 Not Implemented", "Layout is: " + layout) | |||||
class Exiter(object): | class Exiter(object): | ||||
"""App that exits the server, for testing""" | """App that exits the server, for testing""" | ||||
@@ -23,9 +23,13 @@ class Timestamper(object): | |||||
self.file.close() | self.file.close() | ||||
def readline(self, *args): | def readline(self, *args): | ||||
line = self.file.readline(*args) | |||||
if not line: | |||||
return "" | |||||
while True: | |||||
line = self.file.readline(*args) | |||||
if not line: | |||||
return "" | |||||
if line[0] == '#': | |||||
continue | |||||
break | |||||
try: | try: | ||||
return self.ts_iter.next() + line | return self.ts_iter.next() + line | ||||
except StopIteration: | except StopIteration: | ||||
@@ -1,7 +1,7 @@ | |||||
[nosetests] | [nosetests] | ||||
# note: the value doesn't matter, that's why they're empty here | # note: the value doesn't matter, that's why they're empty here | ||||
nocapture= | nocapture= | ||||
nologcapture= # see cherrypy logs on failure | |||||
nologcapture= # comment to see cherrypy logs on failure | |||||
with-coverage= | with-coverage= | ||||
cover-inclusive= | cover-inclusive= | ||||
cover-package=nilmdb | cover-package=nilmdb | ||||
@@ -54,12 +54,12 @@ class TestClient(object): | |||||
def test_client_1_basic(self): | def test_client_1_basic(self): | ||||
# Test a fake host | # Test a fake host | ||||
client = nilmdb.Client(url = "http://localhost:1/") | client = nilmdb.Client(url = "http://localhost:1/") | ||||
with assert_raises(nilmdb.ServerError): | |||||
with assert_raises(nilmdb.client.ServerError): | |||||
client.version() | client.version() | ||||
# Then a fake URL on a real host | # Then a fake URL on a real host | ||||
client = nilmdb.Client(url = "http://localhost:12380/fake/") | client = nilmdb.Client(url = "http://localhost:12380/fake/") | ||||
with assert_raises(nilmdb.ClientError): | |||||
with assert_raises(nilmdb.client.ClientError): | |||||
client.version() | client.version() | ||||
# Now use the real URL | # Now use the real URL | ||||
@@ -26,7 +26,7 @@ class TestTimestamper(object): | |||||
return "\n".join(list) + "\n" | return "\n".join(list) + "\n" | ||||
start = datetime_tz.datetime_tz.smartparse("03/24/2012").totimestamp() | start = datetime_tz.datetime_tz.smartparse("03/24/2012").totimestamp() | ||||
lines_in = [ "hello", "world", "hello world" ] | |||||
lines_in = [ "hello", "world", "hello world", "# commented out" ] | |||||
lines_out = [ "1332561600.000000 hello", | lines_out = [ "1332561600.000000 hello", | ||||
"1332561600.000125 world", | "1332561600.000125 world", | ||||
"1332561600.000250 hello world" ] | "1332561600.000250 hello world" ] | ||||