git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10622 ddd99763-3ecb-0310-9145-efcb8ce7c51ftags/bxinterval-last
@@ -1,6 +1,9 @@ | |||||
all: | all: | ||||
nosetests | nosetests | ||||
profile: | |||||
nosetests --with-profile | |||||
clean:: | clean:: | ||||
find . -name '*pyc' | xargs rm -f | find . -name '*pyc' | xargs rm -f | ||||
rm -f .coverage | rm -f .coverage | ||||
@@ -1,5 +1,5 @@ | |||||
from nilmdb import NilmDB, StreamException | |||||
from nilmdb import NilmDB, StreamError | |||||
from server import Server | from server import Server | ||||
from client import Client | |||||
from client import Client, ClientError | |||||
from layout import * | from layout import * | ||||
from serializer import WrapObject | from serializer import WrapObject |
@@ -8,8 +8,181 @@ import time | |||||
import sys | import sys | ||||
import re | import re | ||||
import os | import os | ||||
import json | |||||
import urlparse | import urlparse | ||||
import urllib2 | |||||
from urllib2 import urlopen, HTTPError | |||||
import pycurl | |||||
class ClientError(Exception): | |||||
def __init__(self, code, message): | |||||
Exception.__init__(self, message) | |||||
self.code = code | |||||
def __str__(self): | |||||
return sprintf("[%03d] %s", self.code, self.message) | |||||
class MyCurl(object): | |||||
def __init__(self, baseurl = ""): | |||||
"""If baseurl is supplied, all other functions that take | |||||
a URL can be given a relative URL instead.""" | |||||
self.baseurl = baseurl | |||||
self.curl = pycurl.Curl() | |||||
self.curl.setopt(pycurl.SSL_VERIFYHOST, 2) | |||||
self.curl.setopt(pycurl.FOLLOWLOCATION, 1) | |||||
self.curl.setopt(pycurl.MAXREDIRS, 5) | |||||
def _setup_url(self, url, params): | |||||
url = urlparse.urljoin(self.baseurl, url) | |||||
if params: | |||||
url = urlparse.urljoin(url, "?" + urllib.urlencode(params, True)) | |||||
self.curl.setopt(pycurl.URL, url) | |||||
return url | |||||
def getjson(self, url, params = None): | |||||
"""Simple GET that returns JSON string""" | |||||
self._setup_url(url, params) | |||||
self._body = "" | |||||
def body_callback(x): | |||||
self._body += x | |||||
self.curl.setopt(pycurl.WRITEFUNCTION, body_callback) | |||||
try: | |||||
self.curl.perform() | |||||
except pycurl.error as e: | |||||
raise ClientError(500, e[1]) | |||||
code = self.curl.getinfo(pycurl.RESPONSE_CODE) | |||||
if code != 200: | |||||
raise ClientError(code, "HTTP error") | |||||
return json.loads(self._body) | |||||
class Client(object): | class Client(object): | ||||
pass | |||||
def __init__(self, url): | |||||
self.curl = MyCurl(url) | |||||
def version(self): | |||||
return self.curl.getjson("version") | |||||
def stream_list(self, path = None, layout = None): | |||||
params = {} | |||||
if path is not None: | |||||
params["path"] = path | |||||
if layout is not None: | |||||
params["layout"] = layout | |||||
return self.curl.getjson("stream/list", params) | |||||
def stream_get_metadata(self, path, keys = None): | |||||
params = { "path": path } | |||||
if keys is not None: | |||||
params["key"] = keys | |||||
return self.curl.getjson("stream/get_metadata", params) | |||||
def stream_set_metadata(self): | |||||
raise NotImplementedError | |||||
def stream_update_metadata(self): | |||||
raise NotImplementedError | |||||
def stream_create(self): | |||||
raise NotImplementedError | |||||
def stream_insert(self, path): | |||||
params = { "path": path } | |||||
return self.curl.getjson("stream/insert", params) | |||||
# eq_(db.stream_list(), []) | |||||
# # Bad path | |||||
# with assert_raises(ValueError): | |||||
# db.stream_create("foo/bar/baz", "PrepData") | |||||
# with assert_raises(ValueError): | |||||
# db.stream_create("/foo", "PrepData") | |||||
# # Bad layout type | |||||
# with assert_raises(KeyError): | |||||
# db.stream_create("/newton/prep", "NoSuchLayout") | |||||
# # Bad index columns | |||||
# with assert_raises(KeyError): | |||||
# db.stream_create("/newton/prep", "PrepData", ["nonexistant"]) | |||||
# db.stream_create("/newton/prep", "PrepData") | |||||
# db.stream_create("/newton/raw", "RawData") | |||||
# db.stream_create("/newton/zzz/rawnotch", "RawNotchedData") | |||||
# # Verify we got 3 streams | |||||
# eq_(db.stream_list(), [ ("/newton/prep", "PrepData"), | |||||
# ("/newton/raw", "RawData"), | |||||
# ("/newton/zzz/rawnotch", "RawNotchedData") | |||||
# ]) | |||||
# # Match just one type | |||||
# eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) | |||||
# # Verify that columns were made right | |||||
# eq_(len(db.h5file.getNode("/newton/prep").cols), 9) | |||||
# eq_(len(db.h5file.getNode("/newton/raw").cols), 7) | |||||
# eq_(len(db.h5file.getNode("/newton/zzz/rawnotch").cols), 10) | |||||
# assert(db.h5file.getNode("/newton/prep").colindexed["timestamp"]) | |||||
# assert(not db.h5file.getNode("/newton/prep").colindexed["p1"]) | |||||
# # Set / get metadata | |||||
# eq_(db.stream_get_metadata("/newton/prep"), {}) | |||||
# eq_(db.stream_get_metadata("/newton/raw"), {}) | |||||
# meta1 = { "description": "The Data", | |||||
# "v_scale": "1.234" } | |||||
# meta2 = { "description": "The Data" } | |||||
# meta3 = { "v_scale": "1.234" } | |||||
# db.stream_set_metadata("/newton/prep", meta1) | |||||
# db.stream_update_metadata("/newton/prep", {}) | |||||
# db.stream_update_metadata("/newton/raw", meta2) | |||||
# db.stream_update_metadata("/newton/raw", meta3) | |||||
# eq_(db.stream_get_metadata("/newton/prep"), meta1) | |||||
# eq_(db.stream_get_metadata("/newton/raw"), meta1) | |||||
# streams = getjson("/stream/list") | |||||
# eq_(streams, [ | |||||
# ['/newton/prep', 'PrepData'], | |||||
# ['/newton/raw', 'RawData'], | |||||
# ['/newton/zzz/rawnotch', 'RawNotchedData'], | |||||
# ]) | |||||
# streams = getjson("/stream/list?layout=RawData") | |||||
# eq_(streams, [['/newton/raw', 'RawData']]) | |||||
# streams = getjson("/stream/list?layout=NoSuchLayout") | |||||
# eq_(streams, []) | |||||
# with assert_raises(HTTPError) as e: | |||||
# getjson("/stream/get_metadata?path=foo") | |||||
# eq_(e.exception.code, 404) | |||||
# data = getjson("/stream/get_metadata?path=/newton/prep") | |||||
# eq_(data, {'description': 'The Data', 'v_scale': '1.234'}) | |||||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||||
# "&key=v_scale") | |||||
# eq_(data, {'v_scale': '1.234'}) | |||||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||||
# "&key=v_scale&key=description") | |||||
# eq_(data, {'description': 'The Data', 'v_scale': '1.234'}) | |||||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||||
# "&key=v_scale&key=foo") | |||||
# eq_(data, {'foo': None, 'v_scale': '1.234'}) | |||||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||||
# "&key=foo") | |||||
# eq_(data, {'foo': None}) | |||||
# def test_insert(self): | |||||
# # invalid path first | |||||
# with assert_raises(HTTPError) as e: | |||||
# getjson("/stream/insert?path=/newton/") | |||||
# eq_(e.exception.code, 404) | |||||
# # XXX TODO | |||||
# with assert_raises(HTTPError) as e: | |||||
# getjson("/stream/insert?path=/newton/prep") | |||||
# eq_(e.exception.code, 501) | |||||
@@ -55,7 +55,7 @@ sql_schema_updates = { | |||||
""", | """, | ||||
} | } | ||||
class StreamException(Exception): | |||||
class StreamError(Exception): | |||||
pass | pass | ||||
class NilmDB(object): | class NilmDB(object): | ||||
@@ -197,7 +197,7 @@ class NilmDB(object): | |||||
result = self.con.execute("SELECT id FROM streams WHERE path=?", | result = self.con.execute("SELECT id FROM streams WHERE path=?", | ||||
(path,)).fetchone() | (path,)).fetchone() | ||||
if result is None: | if result is None: | ||||
raise StreamException("No stream at path " + path) | |||||
raise StreamError("No stream at path " + path) | |||||
return result[0] | return result[0] | ||||
def stream_set_metadata(self, path, data): | def stream_set_metadata(self, path, data): | ||||
@@ -77,7 +77,7 @@ class Stream(NilmApp): | |||||
matching the given keys.""" | matching the given keys.""" | ||||
try: | try: | ||||
data = self.db.stream_get_metadata(path) | data = self.db.stream_get_metadata(path) | ||||
except nilmdb.StreamException as e: | |||||
except nilmdb.StreamError as e: | |||||
raise cherrypy.HTTPError("404 Not Found", e.message) | raise cherrypy.HTTPError("404 Not Found", e.message) | ||||
if key is None: # If no keys specified, return them all | if key is None: # If no keys specified, return them all | ||||
key = data.keys() | key = data.keys() | ||||
@@ -120,7 +120,9 @@ class Exiter(object): | |||||
class Server(object): | class Server(object): | ||||
version = "1.0" | version = "1.0" | ||||
def __init__(self, db, host = '127.0.0.1', port = 8080, stoppable = False, embedded = True): | |||||
def __init__(self, db, host = '127.0.0.1', port = 8080, | |||||
stoppable = False, embedded = True, | |||||
fast_shutdown = False): | |||||
# Need to wrap DB object in a serializer because we'll call into it from separate threads. | # Need to wrap DB object in a serializer because we'll call into it from separate threads. | ||||
self.embedded = embedded | self.embedded = embedded | ||||
self.db = nilmdb.serializer.WrapObject(db) | self.db = nilmdb.serializer.WrapObject(db) | ||||
@@ -138,6 +140,14 @@ class Server(object): | |||||
if stoppable: | if stoppable: | ||||
cherrypy.tree.mount(Exiter(), "/exit") | cherrypy.tree.mount(Exiter(), "/exit") | ||||
# Shutdowns normally wait for clients to disconnect. To speed | |||||
# up tests, set fast_shutdown = True | |||||
if fast_shutdown: | |||||
# Setting timeout to 0 triggers os._exit(70) at shutdown, grr... | |||||
cherrypy.server.shutdown_timeout = 0.01 | |||||
else: | |||||
cherrypy.server.shutdown_timeout = 5 | |||||
def start(self, blocking = False, event = None): | def start(self, blocking = False, event = None): | ||||
if not self.embedded: # pragma: no cover | if not self.embedded: # pragma: no cover | ||||
@@ -10,6 +10,7 @@ stop=1 | |||||
verbosity=2 | verbosity=2 | ||||
#tests=tests/test_nilmdb.py | #tests=tests/test_nilmdb.py | ||||
#tests=tests/test_client.py | #tests=tests/test_client.py | ||||
#with-profile=1 | #with-profile=1 | ||||
profile-sort=time | |||||
profile-restrict=10 | |||||
#profile-sort=time | |||||
#profile-restrict=10 # doesn't work right, treated as string or something |
@@ -1,19 +1,17 @@ | |||||
import nilmdb | import nilmdb | ||||
from nilmdb.printf import * | from nilmdb.printf import * | ||||
from nilmdb.client import ClientError | |||||
from nose.tools import * | from nose.tools import * | ||||
from nose.tools import assert_raises | from nose.tools import assert_raises | ||||
import json | import json | ||||
import itertools | |||||
import itertools | |||||
import distutils.version | |||||
import os | import os | ||||
import shutil | import shutil | ||||
import sys | import sys | ||||
import threading | import threading | ||||
import urllib2 | |||||
from urllib2 import urlopen, HTTPError | |||||
import Queue | |||||
import StringIO | |||||
import shlex | |||||
testdb = "tests/client-testdb" | testdb = "tests/client-testdb" | ||||
@@ -41,7 +39,8 @@ class TestClient(object): | |||||
# Start web app on a custom port | # Start web app on a custom port | ||||
self.db = nilmdb.NilmDB(testdb) | self.db = nilmdb.NilmDB(testdb) | ||||
self.server = nilmdb.Server(self.db, host = "127.0.0.1", | self.server = nilmdb.Server(self.db, host = "127.0.0.1", | ||||
port = 12380, stoppable = False) | |||||
port = 12380, stoppable = False, | |||||
fast_shutdown = True) | |||||
self.server.start(blocking = False) | self.server.start(blocking = False) | ||||
def tearDown(self): | def tearDown(self): | ||||
@@ -52,10 +51,71 @@ class TestClient(object): | |||||
def test_client_basic(self): | def test_client_basic(self): | ||||
# Test a fake host | # Test a fake host | ||||
client = nilmdb.Client(url = "http://localhost:1/") | client = nilmdb.Client(url = "http://localhost:1/") | ||||
with assert_raises(NilmDBError) as e: | |||||
with assert_raises(nilmdb.ClientError) as e: | |||||
client.version() | |||||
eq_(e.exception.code, 500) | |||||
# Then a fake URL on a real host | |||||
client = nilmdb.Client(url = "http://localhost:12380/fake/") | |||||
with assert_raises(nilmdb.ClientError) as e: | |||||
client.version() | client.version() | ||||
eq_(e.exception.code, 404) | |||||
# Now a real host | |||||
# Now use the real URL | |||||
client = nilmdb.Client(url = "http://localhost:12380/") | client = nilmdb.Client(url = "http://localhost:12380/") | ||||
eq_(distutils.version.StrictVersion(getjson("/version")), | |||||
version = client.version() | |||||
eq_(distutils.version.StrictVersion(version), | |||||
distutils.version.StrictVersion(self.server.version)) | distutils.version.StrictVersion(self.server.version)) | ||||
def test_client_nilmdb(self): | |||||
# Basic stream tests, like those in test_nilmdb:test_stream | |||||
client = nilmdb.Client(url = "http://localhost:12380/") | |||||
# Database starts empty | |||||
eq_(client.stream_list(), []) | |||||
# Bad path | |||||
with assert_raises(ClientError): | |||||
client.stream_create("foo/bar/baz", "PrepData") | |||||
with assert_raises(ClientError): | |||||
client.stream_create("/foo", "PrepData") | |||||
# Bad layout type | |||||
with assert_raises(ClientError): | |||||
client.stream_create("/newton/prep", "NoSuchLayout") | |||||
# Bad index columns | |||||
with assert_raises(ClientError): | |||||
client.stream_create("/newton/prep", "PrepData", ["nonexistant"]) | |||||
client.stream_create("/newton/prep", "PrepData") | |||||
client.stream_create("/newton/raw", "RawData") | |||||
client.stream_create("/newton/zzz/rawnotch", "RawNotchedData") | |||||
# Verify we got 3 streams | |||||
eq_(client.stream_list(), [ ("/newton/prep", "PrepData"), | |||||
("/newton/raw", "RawData"), | |||||
("/newton/zzz/rawnotch", "RawNotchedData") | |||||
]) | |||||
# Match just one type or one path | |||||
eq_(client.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) | |||||
eq_(client.stream_list(path="/newton/raw"), [ ("/newton/raw", "RawData") ]) | |||||
# Verify that columns were made right | |||||
eq_(len(client.h5file.getNode("/newton/prep").cols), 9) | |||||
eq_(len(client.h5file.getNode("/newton/raw").cols), 7) | |||||
eq_(len(client.h5file.getNode("/newton/zzz/rawnotch").cols), 10) | |||||
assert(client.h5file.getNode("/newton/prep").colindexed["timestamp"]) | |||||
assert(not client.h5file.getNode("/newton/prep").colindexed["p1"]) | |||||
# Set / get metadata | |||||
eq_(client.stream_get_metadata("/newton/prep"), {}) | |||||
eq_(client.stream_get_metadata("/newton/raw"), {}) | |||||
meta1 = { "description": "The Data", | |||||
"v_scale": "1.234" } | |||||
meta2 = { "description": "The Data" } | |||||
meta3 = { "v_scale": "1.234" } | |||||
client.stream_set_metadata("/newton/prep", meta1) | |||||
client.stream_update_metadata("/newton/prep", {}) | |||||
client.stream_update_metadata("/newton/raw", meta2) | |||||
client.stream_update_metadata("/newton/raw", meta3) | |||||
eq_(client.stream_get_metadata("/newton/prep"), meta1) | |||||
eq_(client.stream_get_metadata("/newton/raw"), meta1) | |||||
@@ -65,14 +65,14 @@ class TestCmdline(object): | |||||
outfile = StringIO.StringIO() | outfile = StringIO.StringIO() | ||||
with stdio_wrapper(infile, outfile, outfile) as s: | with stdio_wrapper(infile, outfile, outfile) as s: | ||||
try: | try: | ||||
nilmdb.client.run(args = shlex.split(arg_string)) | |||||
nilmdb.cmdline.run(args = shlex.split(arg_string)) | |||||
sys.exit(0) | sys.exit(0) | ||||
except SystemExit as e: | except SystemExit as e: | ||||
exitcode = e.code | exitcode = e.code | ||||
output = outfile.getvalue() | output = outfile.getvalue() | ||||
return (output, exitcode) | return (output, exitcode) | ||||
def test_client_basic(self): | |||||
def test_cmdline_basic(self): | |||||
(output, exitcode) = self.run("--help") | (output, exitcode) = self.run("--help") | ||||
assert("Usage:" in output) | assert("Usage:" in output) | ||||
eq_(exitcode, 0) | eq_(exitcode, 0) | ||||
@@ -63,8 +63,9 @@ class Test00Nilmdb(object): # named 00 so it runs first | |||||
("/newton/raw", "RawData"), | ("/newton/raw", "RawData"), | ||||
("/newton/zzz/rawnotch", "RawNotchedData") | ("/newton/zzz/rawnotch", "RawNotchedData") | ||||
]) | ]) | ||||
# Match just one type | |||||
# Match just one type or one path | |||||
eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) | eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) | ||||
eq_(db.stream_list(path="/newton/raw"), [ ("/newton/raw", "RawData") ]) | |||||
# Verify that columns were made right | # Verify that columns were made right | ||||
eq_(len(db.h5file.getNode("/newton/prep").cols), 9) | eq_(len(db.h5file.getNode("/newton/prep").cols), 9) | ||||