Browse Source

pass data to server

git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10655 ddd99763-3ecb-0310-9145-efcb8ce7c51f
tags/bxinterval-last
Jim Paris 12 years ago
parent
commit
e1bf680d13
9 changed files with 40 additions and 34 deletions
  1. +3
    -3
      design.md
  2. +2
    -2
      nilmdb/__init__.py
  3. +12
    -7
      nilmdb/layout.py
  4. +7
    -6
      nilmdb/nilmdb.py
  5. +5
    -9
      nilmdb/server.py
  6. +7
    -3
      nilmdb/timestamper.py
  7. +1
    -1
      setup.cfg
  8. +2
    -2
      tests/test_client.py
  9. +1
    -1
      tests/test_timestamper.py

+ 3
- 3
design.md View File

@@ -74,11 +74,11 @@ Converting from ASCII to PyTables:
- Server converts this ACSII data to a list of values - Server converts this ACSII data to a list of values
- Maybe: - Maybe:
# threaded side creates this object
parser = nilmdb.layout.Parser("layout_name")
# threaded side creates this object
parser = nilmdb.layout.Parser("layout_name")
# threaded side parses and fills it with data # threaded side parses and fills it with data
parser.parse(textdata) parser.parse(textdata)
# serialized side pulls out rows # serialized side pulls out rows
for n in xrange(parser.nrows): for n in xrange(parser.nrows):
parser.fill_row(rowinstance)
parser.fill_row(rowinstance, n)
table.append() table.append()

+ 2
- 2
nilmdb/__init__.py View File

@@ -1,8 +1,8 @@
"""Main NilmDB import""" """Main NilmDB import"""


from .nilmdb import NilmDB, StreamError, OverlapError
from .nilmdb import NilmDB
from .server import Server from .server import Server
from .client import Client, ClientError, ServerError
from .client import Client


import layout import layout
import serializer import serializer


+ 12
- 7
nilmdb/layout.py View File

@@ -72,6 +72,7 @@ class Parser(object):


self.layout = named[layout] self.layout = named[layout]
self.data = [] self.data = []
self.nrows = 0


def parse(self, textdata): def parse(self, textdata):
"""Parse the data, provided as lines of text, using the current """Parse the data, provided as lines of text, using the current
@@ -80,16 +81,20 @@ class Parser(object):
# This currently takes about 0.1 seconds for 1 megabyte of prep data, # This currently takes about 0.1 seconds for 1 megabyte of prep data,
# 85 klines/sec. Could clearly be optimized a lot... # 85 klines/sec. Could clearly be optimized a lot...
indata = cStringIO.StringIO(textdata) indata = cStringIO.StringIO(textdata)
n = 0
self.nrows = 0
# Assume any parsing error is a real error.
# In the future we might want to skip completely empty lines,
# or partial lines right before EOF?
try: try:
for line in indata: for line in indata:
n += 1
self.nrows += 1
fields = line.partition('#')[0].split() fields = line.partition('#')[0].split()
self.data.append(self.layout.parse(fields)) self.data.append(self.layout.parse(fields))
except (ValueError, TypeError, IndexError) as e: except (ValueError, TypeError, IndexError) as e:
raise TypeError("line " + n + ": " + e.message)
raise TypeError("line " + self.nrows + ": " + e.message)


def fillrow(self, row):
"""Fill a PyTables row object that matches the layout with the
stored data"""
def fillrow(self, tablerow, rownum):
"""Fill a PyTables row object with the parsed data.
The row must match the parser's layout"""
for (n, (name, type)) in enumerate(self.layout.fields):
tablerow[name] = self.data[rownum][n]

+ 7
- 6
nilmdb/nilmdb.py View File

@@ -260,12 +260,13 @@ class NilmDB(object):
data.update(newdata) data.update(newdata)
self.stream_set_metadata(path, data) self.stream_set_metadata(path, data)


def stream_insert(self, path, data):
def stream_insert(self, path, parser):
"""Insert new data into the database. """Insert new data into the database.

path: Path of data

data: Data to insert
path: Path at which to add the data
parser: nilmdb.layout.Parser instance full of data to insert
""" """
pass
# First check for basic overlap using timestamp info from the parser.

raise NilmDBError("go away")

+ 5
- 9
nilmdb/server.py View File

@@ -91,7 +91,7 @@ class Stream(NilmApp):
matching the given keys.""" matching the given keys."""
try: try:
data = self.db.stream_get_metadata(path) data = self.db.stream_get_metadata(path)
except nilmdb.StreamError as e:
except nilmdb.nilmdb.StreamError as e:
raise cherrypy.HTTPError("404 Not Found", e.message) raise cherrypy.HTTPError("404 Not Found", e.message)
if key is None: # If no keys specified, return them all if key is None: # If no keys specified, return them all
key = data.keys() key = data.keys()
@@ -169,18 +169,14 @@ class Stream(NilmApp):
raise cherrypy.HTTPError("400 Bad Request", raise cherrypy.HTTPError("400 Bad Request",
"Error parsing input data: " + "Error parsing input data: " +
e.message) e.message)
return "ok"
# XXX TODO
print time.time()
# Do the magic

# Now do the nilmdb insert, passing it the parser full of data.
try: try:
result = self.db.stream_insert(path, body)
except nilmdb.OverlapError as e:
result = self.db.stream_insert(path, parser)
except nilmdb.nilmdb.NilmDBError as e:
raise cherrypy.HTTPError("400 Bad Request", e.message) raise cherrypy.HTTPError("400 Bad Request", e.message)
print time.time()
return "ok" return "ok"
raise cherrypy.HTTPError("501 Not Implemented", "Layout is: " + layout)


class Exiter(object): class Exiter(object):
"""App that exits the server, for testing""" """App that exits the server, for testing"""


+ 7
- 3
nilmdb/timestamper.py View File

@@ -23,9 +23,13 @@ class Timestamper(object):
self.file.close() self.file.close()


def readline(self, *args): def readline(self, *args):
line = self.file.readline(*args)
if not line:
return ""
while True:
line = self.file.readline(*args)
if not line:
return ""
if line[0] == '#':
continue
break
try: try:
return self.ts_iter.next() + line return self.ts_iter.next() + line
except StopIteration: except StopIteration:


+ 1
- 1
setup.cfg View File

@@ -1,7 +1,7 @@
[nosetests] [nosetests]
# note: the value doesn't matter, that's why they're empty here # note: the value doesn't matter, that's why they're empty here
nocapture= nocapture=
nologcapture= # see cherrypy logs on failure
nologcapture= # comment to see cherrypy logs on failure
with-coverage= with-coverage=
cover-inclusive= cover-inclusive=
cover-package=nilmdb cover-package=nilmdb


+ 2
- 2
tests/test_client.py View File

@@ -54,12 +54,12 @@ class TestClient(object):
def test_client_1_basic(self): def test_client_1_basic(self):
# Test a fake host # Test a fake host
client = nilmdb.Client(url = "http://localhost:1/") client = nilmdb.Client(url = "http://localhost:1/")
with assert_raises(nilmdb.ServerError):
with assert_raises(nilmdb.client.ServerError):
client.version() client.version()


# Then a fake URL on a real host # Then a fake URL on a real host
client = nilmdb.Client(url = "http://localhost:12380/fake/") client = nilmdb.Client(url = "http://localhost:12380/fake/")
with assert_raises(nilmdb.ClientError):
with assert_raises(nilmdb.client.ClientError):
client.version() client.version()


# Now use the real URL # Now use the real URL


+ 1
- 1
tests/test_timestamper.py View File

@@ -26,7 +26,7 @@ class TestTimestamper(object):
return "\n".join(list) + "\n" return "\n".join(list) + "\n"


start = datetime_tz.datetime_tz.smartparse("03/24/2012").totimestamp() start = datetime_tz.datetime_tz.smartparse("03/24/2012").totimestamp()
lines_in = [ "hello", "world", "hello world" ]
lines_in = [ "hello", "world", "hello world", "# commented out" ]
lines_out = [ "1332561600.000000 hello", lines_out = [ "1332561600.000000 hello",
"1332561600.000125 world", "1332561600.000125 world",
"1332561600.000250 hello world" ] "1332561600.000250 hello world" ]


Loading…
Cancel
Save