Browse Source

rename command line client nilmdb.py to nilmtool.py

git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10622 ddd99763-3ecb-0310-9145-efcb8ce7c51f
tags/bxinterval-last
Jim Paris 12 years ago
parent
commit
1c4efb92c6
10 changed files with 270 additions and 22 deletions
  1. +3
    -0
      Makefile
  2. +2
    -2
      nilmdb/__init__.py
  3. +174
    -1
      nilmdb/client.py
  4. +2
    -2
      nilmdb/nilmdb.py
  5. +12
    -2
      nilmdb/server.py
  6. +0
    -0
      nilmtool.py
  7. +3
    -2
      setup.cfg
  8. +70
    -10
      tests/test_client.py
  9. +2
    -2
      tests/test_cmdline.py
  10. +2
    -1
      tests/test_nilmdb.py

+ 3
- 0
Makefile View File

@@ -1,6 +1,9 @@
all: all:
nosetests nosetests


profile:
nosetests --with-profile

clean:: clean::
find . -name '*pyc' | xargs rm -f find . -name '*pyc' | xargs rm -f
rm -f .coverage rm -f .coverage


+ 2
- 2
nilmdb/__init__.py View File

@@ -1,5 +1,5 @@
from nilmdb import NilmDB, StreamException
from nilmdb import NilmDB, StreamError
from server import Server from server import Server
from client import Client
from client import Client, ClientError
from layout import * from layout import *
from serializer import WrapObject from serializer import WrapObject

+ 174
- 1
nilmdb/client.py View File

@@ -8,8 +8,181 @@ import time
import sys import sys
import re import re
import os import os
import json
import urlparse import urlparse
import urllib2
from urllib2 import urlopen, HTTPError
import pycurl

class ClientError(Exception):
def __init__(self, code, message):
Exception.__init__(self, message)
self.code = code
def __str__(self):
return sprintf("[%03d] %s", self.code, self.message)

class MyCurl(object):
def __init__(self, baseurl = ""):
"""If baseurl is supplied, all other functions that take
a URL can be given a relative URL instead."""
self.baseurl = baseurl
self.curl = pycurl.Curl()
self.curl.setopt(pycurl.SSL_VERIFYHOST, 2)
self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
self.curl.setopt(pycurl.MAXREDIRS, 5)

def _setup_url(self, url, params):
url = urlparse.urljoin(self.baseurl, url)
if params:
url = urlparse.urljoin(url, "?" + urllib.urlencode(params, True))
self.curl.setopt(pycurl.URL, url)
return url
def getjson(self, url, params = None):
"""Simple GET that returns JSON string"""
self._setup_url(url, params)
self._body = ""
def body_callback(x):
self._body += x
self.curl.setopt(pycurl.WRITEFUNCTION, body_callback)
try:
self.curl.perform()
except pycurl.error as e:
raise ClientError(500, e[1])
code = self.curl.getinfo(pycurl.RESPONSE_CODE)
if code != 200:
raise ClientError(code, "HTTP error")
return json.loads(self._body)


class Client(object): class Client(object):
pass
def __init__(self, url):
self.curl = MyCurl(url)

def version(self):
return self.curl.getjson("version")

def stream_list(self, path = None, layout = None):
params = {}
if path is not None:
params["path"] = path
if layout is not None:
params["layout"] = layout
return self.curl.getjson("stream/list", params)

def stream_get_metadata(self, path, keys = None):
params = { "path": path }
if keys is not None:
params["key"] = keys
return self.curl.getjson("stream/get_metadata", params)

def stream_set_metadata(self):
raise NotImplementedError

def stream_update_metadata(self):
raise NotImplementedError

def stream_create(self):
raise NotImplementedError

def stream_insert(self, path):
params = { "path": path }
return self.curl.getjson("stream/insert", params)

# eq_(db.stream_list(), [])

# # Bad path
# with assert_raises(ValueError):
# db.stream_create("foo/bar/baz", "PrepData")
# with assert_raises(ValueError):
# db.stream_create("/foo", "PrepData")
# # Bad layout type
# with assert_raises(KeyError):
# db.stream_create("/newton/prep", "NoSuchLayout")
# # Bad index columns
# with assert_raises(KeyError):
# db.stream_create("/newton/prep", "PrepData", ["nonexistant"])
# db.stream_create("/newton/prep", "PrepData")
# db.stream_create("/newton/raw", "RawData")
# db.stream_create("/newton/zzz/rawnotch", "RawNotchedData")

# # Verify we got 3 streams
# eq_(db.stream_list(), [ ("/newton/prep", "PrepData"),
# ("/newton/raw", "RawData"),
# ("/newton/zzz/rawnotch", "RawNotchedData")
# ])
# # Match just one type
# eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ])

# # Verify that columns were made right
# eq_(len(db.h5file.getNode("/newton/prep").cols), 9)
# eq_(len(db.h5file.getNode("/newton/raw").cols), 7)
# eq_(len(db.h5file.getNode("/newton/zzz/rawnotch").cols), 10)
# assert(db.h5file.getNode("/newton/prep").colindexed["timestamp"])
# assert(not db.h5file.getNode("/newton/prep").colindexed["p1"])

# # Set / get metadata
# eq_(db.stream_get_metadata("/newton/prep"), {})
# eq_(db.stream_get_metadata("/newton/raw"), {})
# meta1 = { "description": "The Data",
# "v_scale": "1.234" }
# meta2 = { "description": "The Data" }
# meta3 = { "v_scale": "1.234" }
# db.stream_set_metadata("/newton/prep", meta1)
# db.stream_update_metadata("/newton/prep", {})
# db.stream_update_metadata("/newton/raw", meta2)
# db.stream_update_metadata("/newton/raw", meta3)
# eq_(db.stream_get_metadata("/newton/prep"), meta1)
# eq_(db.stream_get_metadata("/newton/raw"), meta1)


# streams = getjson("/stream/list")

# eq_(streams, [
# ['/newton/prep', 'PrepData'],
# ['/newton/raw', 'RawData'],
# ['/newton/zzz/rawnotch', 'RawNotchedData'],
# ])

# streams = getjson("/stream/list?layout=RawData")
# eq_(streams, [['/newton/raw', 'RawData']])

# streams = getjson("/stream/list?layout=NoSuchLayout")
# eq_(streams, [])

# with assert_raises(HTTPError) as e:
# getjson("/stream/get_metadata?path=foo")
# eq_(e.exception.code, 404)

# data = getjson("/stream/get_metadata?path=/newton/prep")
# eq_(data, {'description': 'The Data', 'v_scale': '1.234'})

# data = getjson("/stream/get_metadata?path=/newton/prep"
# "&key=v_scale")
# eq_(data, {'v_scale': '1.234'})

# data = getjson("/stream/get_metadata?path=/newton/prep"
# "&key=v_scale&key=description")
# eq_(data, {'description': 'The Data', 'v_scale': '1.234'})

# data = getjson("/stream/get_metadata?path=/newton/prep"
# "&key=v_scale&key=foo")
# eq_(data, {'foo': None, 'v_scale': '1.234'})

# data = getjson("/stream/get_metadata?path=/newton/prep"
# "&key=foo")
# eq_(data, {'foo': None})


# def test_insert(self):
# # invalid path first
# with assert_raises(HTTPError) as e:
# getjson("/stream/insert?path=/newton/")
# eq_(e.exception.code, 404)

# # XXX TODO
# with assert_raises(HTTPError) as e:
# getjson("/stream/insert?path=/newton/prep")
# eq_(e.exception.code, 501)





+ 2
- 2
nilmdb/nilmdb.py View File

@@ -55,7 +55,7 @@ sql_schema_updates = {
""", """,
} }


class StreamException(Exception):
class StreamError(Exception):
pass pass


class NilmDB(object): class NilmDB(object):
@@ -197,7 +197,7 @@ class NilmDB(object):
result = self.con.execute("SELECT id FROM streams WHERE path=?", result = self.con.execute("SELECT id FROM streams WHERE path=?",
(path,)).fetchone() (path,)).fetchone()
if result is None: if result is None:
raise StreamException("No stream at path " + path)
raise StreamError("No stream at path " + path)
return result[0] return result[0]
def stream_set_metadata(self, path, data): def stream_set_metadata(self, path, data):


+ 12
- 2
nilmdb/server.py View File

@@ -77,7 +77,7 @@ class Stream(NilmApp):
matching the given keys.""" matching the given keys."""
try: try:
data = self.db.stream_get_metadata(path) data = self.db.stream_get_metadata(path)
except nilmdb.StreamException as e:
except nilmdb.StreamError as e:
raise cherrypy.HTTPError("404 Not Found", e.message) raise cherrypy.HTTPError("404 Not Found", e.message)
if key is None: # If no keys specified, return them all if key is None: # If no keys specified, return them all
key = data.keys() key = data.keys()
@@ -120,7 +120,9 @@ class Exiter(object):
class Server(object): class Server(object):
version = "1.0" version = "1.0"
def __init__(self, db, host = '127.0.0.1', port = 8080, stoppable = False, embedded = True):
def __init__(self, db, host = '127.0.0.1', port = 8080,
stoppable = False, embedded = True,
fast_shutdown = False):
# Need to wrap DB object in a serializer because we'll call into it from separate threads. # Need to wrap DB object in a serializer because we'll call into it from separate threads.
self.embedded = embedded self.embedded = embedded
self.db = nilmdb.serializer.WrapObject(db) self.db = nilmdb.serializer.WrapObject(db)
@@ -138,6 +140,14 @@ class Server(object):
if stoppable: if stoppable:
cherrypy.tree.mount(Exiter(), "/exit") cherrypy.tree.mount(Exiter(), "/exit")


# Shutdowns normally wait for clients to disconnect. To speed
# up tests, set fast_shutdown = True
if fast_shutdown:
# Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
cherrypy.server.shutdown_timeout = 0.01
else:
cherrypy.server.shutdown_timeout = 5

def start(self, blocking = False, event = None): def start(self, blocking = False, event = None):


if not self.embedded: # pragma: no cover if not self.embedded: # pragma: no cover


nilmdb.py → nilmtool.py View File


+ 3
- 2
setup.cfg View File

@@ -10,6 +10,7 @@ stop=1
verbosity=2 verbosity=2
#tests=tests/test_nilmdb.py #tests=tests/test_nilmdb.py
#tests=tests/test_client.py #tests=tests/test_client.py

#with-profile=1 #with-profile=1
profile-sort=time
profile-restrict=10
#profile-sort=time
#profile-restrict=10 # doesn't work right, treated as string or something

+ 70
- 10
tests/test_client.py View File

@@ -1,19 +1,17 @@
import nilmdb import nilmdb
from nilmdb.printf import * from nilmdb.printf import *


from nilmdb.client import ClientError

from nose.tools import * from nose.tools import *
from nose.tools import assert_raises from nose.tools import assert_raises
import json import json
import itertools
import itertools
import distutils.version
import os import os
import shutil import shutil
import sys import sys
import threading import threading
import urllib2
from urllib2 import urlopen, HTTPError
import Queue
import StringIO
import shlex


testdb = "tests/client-testdb" testdb = "tests/client-testdb"


@@ -41,7 +39,8 @@ class TestClient(object):
# Start web app on a custom port # Start web app on a custom port
self.db = nilmdb.NilmDB(testdb) self.db = nilmdb.NilmDB(testdb)
self.server = nilmdb.Server(self.db, host = "127.0.0.1", self.server = nilmdb.Server(self.db, host = "127.0.0.1",
port = 12380, stoppable = False)
port = 12380, stoppable = False,
fast_shutdown = True)
self.server.start(blocking = False) self.server.start(blocking = False)


def tearDown(self): def tearDown(self):
@@ -52,10 +51,71 @@ class TestClient(object):
def test_client_basic(self): def test_client_basic(self):
# Test a fake host # Test a fake host
client = nilmdb.Client(url = "http://localhost:1/") client = nilmdb.Client(url = "http://localhost:1/")
with assert_raises(NilmDBError) as e:
with assert_raises(nilmdb.ClientError) as e:
client.version()
eq_(e.exception.code, 500)

# Then a fake URL on a real host
client = nilmdb.Client(url = "http://localhost:12380/fake/")
with assert_raises(nilmdb.ClientError) as e:
client.version() client.version()
eq_(e.exception.code, 404)


# Now a real host
# Now use the real URL
client = nilmdb.Client(url = "http://localhost:12380/") client = nilmdb.Client(url = "http://localhost:12380/")
eq_(distutils.version.StrictVersion(getjson("/version")),
version = client.version()
eq_(distutils.version.StrictVersion(version),
distutils.version.StrictVersion(self.server.version)) distutils.version.StrictVersion(self.server.version))

def test_client_nilmdb(self):
# Basic stream tests, like those in test_nilmdb:test_stream
client = nilmdb.Client(url = "http://localhost:12380/")

# Database starts empty
eq_(client.stream_list(), [])

# Bad path
with assert_raises(ClientError):
client.stream_create("foo/bar/baz", "PrepData")
with assert_raises(ClientError):
client.stream_create("/foo", "PrepData")
# Bad layout type
with assert_raises(ClientError):
client.stream_create("/newton/prep", "NoSuchLayout")
# Bad index columns
with assert_raises(ClientError):
client.stream_create("/newton/prep", "PrepData", ["nonexistant"])
client.stream_create("/newton/prep", "PrepData")
client.stream_create("/newton/raw", "RawData")
client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")

# Verify we got 3 streams
eq_(client.stream_list(), [ ("/newton/prep", "PrepData"),
("/newton/raw", "RawData"),
("/newton/zzz/rawnotch", "RawNotchedData")
])
# Match just one type or one path
eq_(client.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ])
eq_(client.stream_list(path="/newton/raw"), [ ("/newton/raw", "RawData") ])

# Verify that columns were made right
eq_(len(client.h5file.getNode("/newton/prep").cols), 9)
eq_(len(client.h5file.getNode("/newton/raw").cols), 7)
eq_(len(client.h5file.getNode("/newton/zzz/rawnotch").cols), 10)
assert(client.h5file.getNode("/newton/prep").colindexed["timestamp"])
assert(not client.h5file.getNode("/newton/prep").colindexed["p1"])

# Set / get metadata
eq_(client.stream_get_metadata("/newton/prep"), {})
eq_(client.stream_get_metadata("/newton/raw"), {})
meta1 = { "description": "The Data",
"v_scale": "1.234" }
meta2 = { "description": "The Data" }
meta3 = { "v_scale": "1.234" }
client.stream_set_metadata("/newton/prep", meta1)
client.stream_update_metadata("/newton/prep", {})
client.stream_update_metadata("/newton/raw", meta2)
client.stream_update_metadata("/newton/raw", meta3)
eq_(client.stream_get_metadata("/newton/prep"), meta1)
eq_(client.stream_get_metadata("/newton/raw"), meta1)


+ 2
- 2
tests/test_cmdline.py View File

@@ -65,14 +65,14 @@ class TestCmdline(object):
outfile = StringIO.StringIO() outfile = StringIO.StringIO()
with stdio_wrapper(infile, outfile, outfile) as s: with stdio_wrapper(infile, outfile, outfile) as s:
try: try:
nilmdb.client.run(args = shlex.split(arg_string))
nilmdb.cmdline.run(args = shlex.split(arg_string))
sys.exit(0) sys.exit(0)
except SystemExit as e: except SystemExit as e:
exitcode = e.code exitcode = e.code
output = outfile.getvalue() output = outfile.getvalue()
return (output, exitcode) return (output, exitcode)


def test_client_basic(self):
def test_cmdline_basic(self):
(output, exitcode) = self.run("--help") (output, exitcode) = self.run("--help")
assert("Usage:" in output) assert("Usage:" in output)
eq_(exitcode, 0) eq_(exitcode, 0)


+ 2
- 1
tests/test_nilmdb.py View File

@@ -63,8 +63,9 @@ class Test00Nilmdb(object): # named 00 so it runs first
("/newton/raw", "RawData"), ("/newton/raw", "RawData"),
("/newton/zzz/rawnotch", "RawNotchedData") ("/newton/zzz/rawnotch", "RawNotchedData")
]) ])
# Match just one type
# Match just one type or one path
eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ]) eq_(db.stream_list(layout="RawData"), [ ("/newton/raw", "RawData") ])
eq_(db.stream_list(path="/newton/raw"), [ ("/newton/raw", "RawData") ])


# Verify that columns were made right # Verify that columns were made right
eq_(len(db.h5file.getNode("/newton/prep").cols), 9) eq_(len(db.h5file.getNode("/newton/prep").cols), 9)


Loading…
Cancel
Save