git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10622 ddd99763-3ecb-0310-9145-efcb8ce7c51ftags/bxinterval-last
@@ -1,6 +1,9 @@ | |||
all: | |||
nosetests | |||
profile: | |||
nosetests --with-profile | |||
clean:: | |||
find . -name '*pyc' | xargs rm -f | |||
rm -f .coverage | |||
@@ -1,5 +1,5 @@ | |||
from nilmdb import NilmDB, StreamException | |||
from nilmdb import NilmDB, StreamError | |||
from server import Server | |||
from client import Client | |||
from client import Client, ClientError | |||
from layout import * | |||
from serializer import WrapObject |
@@ -8,8 +8,181 @@ import time | |||
import sys | |||
import re | |||
import os | |||
import json | |||
import urlparse | |||
import urllib2 | |||
from urllib2 import urlopen, HTTPError | |||
import pycurl | |||
class ClientError(Exception): | |||
def __init__(self, code, message): | |||
Exception.__init__(self, message) | |||
self.code = code | |||
def __str__(self): | |||
return sprintf("[%03d] %s", self.code, self.message) | |||
class MyCurl(object): | |||
def __init__(self, baseurl = ""): | |||
"""If baseurl is supplied, all other functions that take | |||
a URL can be given a relative URL instead.""" | |||
self.baseurl = baseurl | |||
self.curl = pycurl.Curl() | |||
self.curl.setopt(pycurl.SSL_VERIFYHOST, 2) | |||
self.curl.setopt(pycurl.FOLLOWLOCATION, 1) | |||
self.curl.setopt(pycurl.MAXREDIRS, 5) | |||
def _setup_url(self, url, params): | |||
url = urlparse.urljoin(self.baseurl, url) | |||
if params: | |||
url = urlparse.urljoin(url, "?" + urllib.urlencode(params, True)) | |||
self.curl.setopt(pycurl.URL, url) | |||
return url | |||
def getjson(self, url, params = None): | |||
"""Simple GET that returns JSON string""" | |||
self._setup_url(url, params) | |||
self._body = "" | |||
def body_callback(x): | |||
self._body += x | |||
self.curl.setopt(pycurl.WRITEFUNCTION, body_callback) | |||
try: | |||
self.curl.perform() | |||
except pycurl.error as e: | |||
raise ClientError(500, e[1]) | |||
code = self.curl.getinfo(pycurl.RESPONSE_CODE) | |||
if code != 200: | |||
raise ClientError(code, "HTTP error") | |||
return json.loads(self._body) | |||
class Client(object): | |||
pass | |||
def __init__(self, url): | |||
self.curl = MyCurl(url) | |||
def version(self): | |||
return self.curl.getjson("version") | |||
def stream_list(self, path = None, layout = None): | |||
params = {} | |||
if path is not None: | |||
params["path"] = path | |||
if layout is not None: | |||
params["layout"] = layout | |||
return self.curl.getjson("stream/list", params) | |||
def stream_get_metadata(self, path, keys = None): | |||
params = { "path": path } | |||
if keys is not None: | |||
params["key"] = keys | |||
return self.curl.getjson("stream/get_metadata", params) | |||
def stream_set_metadata(self): | |||
raise NotImplementedError | |||
def stream_update_metadata(self): | |||
raise NotImplementedError | |||
def stream_create(self): | |||
raise NotImplementedError | |||
def stream_insert(self, path): | |||
params = { "path": path } | |||
return self.curl.getjson("stream/insert", params) | |||
# eq_(db.stream_list(), []) | |||
# # Bad path | |||
# with assert_raises(ValueError): | |||
# db.stream_create("foo/bar/baz", "PrepData") | |||
# with assert_raises(ValueError): | |||
# db.stream_create("/foo", "PrepData") | |||
# # Bad layout type | |||
# with assert_raises(KeyError): | |||
# db.stream_create("/newton/prep", "NoSuchLayout") | |||
# # Bad index columns | |||
# with assert_raises(KeyError): | |||
# db.stream_create("/newton/prep", "PrepData", ["nonexistant"]) | |||
# db.stream_create("/newton/prep", "PrepData") | |||
# db.stream_create("/newton/raw", "RawData") | |||
# db.stream_create("/newton/zzz/rawnotch", "RawNotchedData") | |||
# # Verify we got 3 streams | |||
# eq_(db.stream_list(), [ ("/newton/prep", "PrepData"), | |||
# ("/newton/raw", "RawData"), | |||
# ("/newton/zzz/rawnotch", "RawNotchedData") | |||
# ]) | |||
# # Match just one type | |||
# eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) | |||
# # Verify that columns were made right | |||
# eq_(len(db.h5file.getNode("/newton/prep").cols), 9) | |||
# eq_(len(db.h5file.getNode("/newton/raw").cols), 7) | |||
# eq_(len(db.h5file.getNode("/newton/zzz/rawnotch").cols), 10) | |||
# assert(db.h5file.getNode("/newton/prep").colindexed["timestamp"]) | |||
# assert(not db.h5file.getNode("/newton/prep").colindexed["p1"]) | |||
# # Set / get metadata | |||
# eq_(db.stream_get_metadata("/newton/prep"), {}) | |||
# eq_(db.stream_get_metadata("/newton/raw"), {}) | |||
# meta1 = { "description": "The Data", | |||
# "v_scale": "1.234" } | |||
# meta2 = { "description": "The Data" } | |||
# meta3 = { "v_scale": "1.234" } | |||
# db.stream_set_metadata("/newton/prep", meta1) | |||
# db.stream_update_metadata("/newton/prep", {}) | |||
# db.stream_update_metadata("/newton/raw", meta2) | |||
# db.stream_update_metadata("/newton/raw", meta3) | |||
# eq_(db.stream_get_metadata("/newton/prep"), meta1) | |||
# eq_(db.stream_get_metadata("/newton/raw"), meta1) | |||
# streams = getjson("/stream/list") | |||
# eq_(streams, [ | |||
# ['/newton/prep', 'PrepData'], | |||
# ['/newton/raw', 'RawData'], | |||
# ['/newton/zzz/rawnotch', 'RawNotchedData'], | |||
# ]) | |||
# streams = getjson("/stream/list?layout=RawData") | |||
# eq_(streams, [['/newton/raw', 'RawData']]) | |||
# streams = getjson("/stream/list?layout=NoSuchLayout") | |||
# eq_(streams, []) | |||
# with assert_raises(HTTPError) as e: | |||
# getjson("/stream/get_metadata?path=foo") | |||
# eq_(e.exception.code, 404) | |||
# data = getjson("/stream/get_metadata?path=/newton/prep") | |||
# eq_(data, {'description': 'The Data', 'v_scale': '1.234'}) | |||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||
# "&key=v_scale") | |||
# eq_(data, {'v_scale': '1.234'}) | |||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||
# "&key=v_scale&key=description") | |||
# eq_(data, {'description': 'The Data', 'v_scale': '1.234'}) | |||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||
# "&key=v_scale&key=foo") | |||
# eq_(data, {'foo': None, 'v_scale': '1.234'}) | |||
# data = getjson("/stream/get_metadata?path=/newton/prep" | |||
# "&key=foo") | |||
# eq_(data, {'foo': None}) | |||
# def test_insert(self): | |||
# # invalid path first | |||
# with assert_raises(HTTPError) as e: | |||
# getjson("/stream/insert?path=/newton/") | |||
# eq_(e.exception.code, 404) | |||
# # XXX TODO | |||
# with assert_raises(HTTPError) as e: | |||
# getjson("/stream/insert?path=/newton/prep") | |||
# eq_(e.exception.code, 501) | |||
@@ -55,7 +55,7 @@ sql_schema_updates = { | |||
""", | |||
} | |||
class StreamException(Exception): | |||
class StreamError(Exception): | |||
pass | |||
class NilmDB(object): | |||
@@ -197,7 +197,7 @@ class NilmDB(object): | |||
result = self.con.execute("SELECT id FROM streams WHERE path=?", | |||
(path,)).fetchone() | |||
if result is None: | |||
raise StreamException("No stream at path " + path) | |||
raise StreamError("No stream at path " + path) | |||
return result[0] | |||
def stream_set_metadata(self, path, data): | |||
@@ -77,7 +77,7 @@ class Stream(NilmApp): | |||
matching the given keys.""" | |||
try: | |||
data = self.db.stream_get_metadata(path) | |||
except nilmdb.StreamException as e: | |||
except nilmdb.StreamError as e: | |||
raise cherrypy.HTTPError("404 Not Found", e.message) | |||
if key is None: # If no keys specified, return them all | |||
key = data.keys() | |||
@@ -120,7 +120,9 @@ class Exiter(object): | |||
class Server(object): | |||
version = "1.0" | |||
def __init__(self, db, host = '127.0.0.1', port = 8080, stoppable = False, embedded = True): | |||
def __init__(self, db, host = '127.0.0.1', port = 8080, | |||
stoppable = False, embedded = True, | |||
fast_shutdown = False): | |||
# Need to wrap DB object in a serializer because we'll call into it from separate threads. | |||
self.embedded = embedded | |||
self.db = nilmdb.serializer.WrapObject(db) | |||
@@ -138,6 +140,14 @@ class Server(object): | |||
if stoppable: | |||
cherrypy.tree.mount(Exiter(), "/exit") | |||
# Shutdowns normally wait for clients to disconnect. To speed | |||
# up tests, set fast_shutdown = True | |||
if fast_shutdown: | |||
# Setting timeout to 0 triggers os._exit(70) at shutdown, grr... | |||
cherrypy.server.shutdown_timeout = 0.01 | |||
else: | |||
cherrypy.server.shutdown_timeout = 5 | |||
def start(self, blocking = False, event = None): | |||
if not self.embedded: # pragma: no cover | |||
@@ -10,6 +10,7 @@ stop=1 | |||
verbosity=2 | |||
#tests=tests/test_nilmdb.py | |||
#tests=tests/test_client.py | |||
#with-profile=1 | |||
profile-sort=time | |||
profile-restrict=10 | |||
#profile-sort=time | |||
#profile-restrict=10 # doesn't work right, treated as string or something |
@@ -1,19 +1,17 @@ | |||
import nilmdb | |||
from nilmdb.printf import * | |||
from nilmdb.client import ClientError | |||
from nose.tools import * | |||
from nose.tools import assert_raises | |||
import json | |||
import itertools | |||
import itertools | |||
import distutils.version | |||
import os | |||
import shutil | |||
import sys | |||
import threading | |||
import urllib2 | |||
from urllib2 import urlopen, HTTPError | |||
import Queue | |||
import StringIO | |||
import shlex | |||
testdb = "tests/client-testdb" | |||
@@ -41,7 +39,8 @@ class TestClient(object): | |||
# Start web app on a custom port | |||
self.db = nilmdb.NilmDB(testdb) | |||
self.server = nilmdb.Server(self.db, host = "127.0.0.1", | |||
port = 12380, stoppable = False) | |||
port = 12380, stoppable = False, | |||
fast_shutdown = True) | |||
self.server.start(blocking = False) | |||
def tearDown(self): | |||
@@ -52,10 +51,71 @@ class TestClient(object): | |||
def test_client_basic(self): | |||
# Test a fake host | |||
client = nilmdb.Client(url = "http://localhost:1/") | |||
with assert_raises(NilmDBError) as e: | |||
with assert_raises(nilmdb.ClientError) as e: | |||
client.version() | |||
eq_(e.exception.code, 500) | |||
# Then a fake URL on a real host | |||
client = nilmdb.Client(url = "http://localhost:12380/fake/") | |||
with assert_raises(nilmdb.ClientError) as e: | |||
client.version() | |||
eq_(e.exception.code, 404) | |||
# Now a real host | |||
# Now use the real URL | |||
client = nilmdb.Client(url = "http://localhost:12380/") | |||
eq_(distutils.version.StrictVersion(getjson("/version")), | |||
version = client.version() | |||
eq_(distutils.version.StrictVersion(version), | |||
distutils.version.StrictVersion(self.server.version)) | |||
def test_client_nilmdb(self): | |||
# Basic stream tests, like those in test_nilmdb:test_stream | |||
client = nilmdb.Client(url = "http://localhost:12380/") | |||
# Database starts empty | |||
eq_(client.stream_list(), []) | |||
# Bad path | |||
with assert_raises(ClientError): | |||
client.stream_create("foo/bar/baz", "PrepData") | |||
with assert_raises(ClientError): | |||
client.stream_create("/foo", "PrepData") | |||
# Bad layout type | |||
with assert_raises(ClientError): | |||
client.stream_create("/newton/prep", "NoSuchLayout") | |||
# Bad index columns | |||
with assert_raises(ClientError): | |||
client.stream_create("/newton/prep", "PrepData", ["nonexistant"]) | |||
client.stream_create("/newton/prep", "PrepData") | |||
client.stream_create("/newton/raw", "RawData") | |||
client.stream_create("/newton/zzz/rawnotch", "RawNotchedData") | |||
# Verify we got 3 streams | |||
eq_(client.stream_list(), [ ("/newton/prep", "PrepData"), | |||
("/newton/raw", "RawData"), | |||
("/newton/zzz/rawnotch", "RawNotchedData") | |||
]) | |||
# Match just one type or one path | |||
eq_(client.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) | |||
eq_(client.stream_list(path="/newton/raw"), [ ("/newton/raw", "RawData") ]) | |||
# Verify that columns were made right | |||
eq_(len(client.h5file.getNode("/newton/prep").cols), 9) | |||
eq_(len(client.h5file.getNode("/newton/raw").cols), 7) | |||
eq_(len(client.h5file.getNode("/newton/zzz/rawnotch").cols), 10) | |||
assert(client.h5file.getNode("/newton/prep").colindexed["timestamp"]) | |||
assert(not client.h5file.getNode("/newton/prep").colindexed["p1"]) | |||
# Set / get metadata | |||
eq_(client.stream_get_metadata("/newton/prep"), {}) | |||
eq_(client.stream_get_metadata("/newton/raw"), {}) | |||
meta1 = { "description": "The Data", | |||
"v_scale": "1.234" } | |||
meta2 = { "description": "The Data" } | |||
meta3 = { "v_scale": "1.234" } | |||
client.stream_set_metadata("/newton/prep", meta1) | |||
client.stream_update_metadata("/newton/prep", {}) | |||
client.stream_update_metadata("/newton/raw", meta2) | |||
client.stream_update_metadata("/newton/raw", meta3) | |||
eq_(client.stream_get_metadata("/newton/prep"), meta1) | |||
eq_(client.stream_get_metadata("/newton/raw"), meta1) | |||
@@ -65,14 +65,14 @@ class TestCmdline(object): | |||
outfile = StringIO.StringIO() | |||
with stdio_wrapper(infile, outfile, outfile) as s: | |||
try: | |||
nilmdb.client.run(args = shlex.split(arg_string)) | |||
nilmdb.cmdline.run(args = shlex.split(arg_string)) | |||
sys.exit(0) | |||
except SystemExit as e: | |||
exitcode = e.code | |||
output = outfile.getvalue() | |||
return (output, exitcode) | |||
def test_client_basic(self): | |||
def test_cmdline_basic(self): | |||
(output, exitcode) = self.run("--help") | |||
assert("Usage:" in output) | |||
eq_(exitcode, 0) | |||
@@ -63,8 +63,9 @@ class Test00Nilmdb(object): # named 00 so it runs first | |||
("/newton/raw", "RawData"), | |||
("/newton/zzz/rawnotch", "RawNotchedData") | |||
]) | |||
# Match just one type | |||
# Match just one type or one path | |||
eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) | |||
eq_(db.stream_list(path="/newton/raw"), [ ("/newton/raw", "RawData") ]) | |||
# Verify that columns were made right | |||
eq_(len(db.h5file.getNode("/newton/prep").cols), 9) | |||