nilm/nilmdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

A lot more command line testing.

Browse Source 
There'es some issue with tons of requests getting slowly blocked,
though...



git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10678 ddd99763-3ecb-0310-9145-efcb8ce7c51f
This commit is contained in:
Jim Paris 2012-04-04 22:34:01 +00:00
parent 277b0c1d00
commit a3f444eb25
6 changed files with 284 additions and 39 deletions

113
nilmdb/cmdline.py
View File

@ -11,6 +11,7 @@ import re
import os
import urlparse
import argparse
import fnmatch
version = "0.1"
@ -32,8 +33,6 @@ class Cmdline(object):
help='show this help message and exit')
group.add_argument("-V", "--version", action="version",
version=version_string)
group.add_argument("-q", "--quiet", action="store_true",
help="suppress unnecessary console output")
group = parser.add_argument_group("Server")
group.add_argument("-u", "--url", action="store",
@ -52,31 +51,64 @@ class Cmdline(object):
List information about the server, like
version.
""")
cmd.set_defaults(handler = self.cmd_info)
# list
cmd = sub.add_parser("list", help="List streams",
formatter_class = formatter,
description="""
List streams available in the database,
optionally filtering by type or partial path.
optionally filtering by type or path. Wildcards
are accepted.
""")
cmd.set_defaults(handler = self.cmd_list)
group = cmd.add_argument_group("Stream filtering")
group.add_argument("-t", "--type", metavar="GLOB", default="*",
group.add_argument("-t", "--type", default="*",
help="Match only this stream type")
group.add_argument("-p", "--path", metavar="GLOB", default="*",
group.add_argument("-p", "--path", default="*",
help="Match only this path")
# group.add_argument(
# create
cmd = sub.add_parser("create", help="Create a new stream",
formatter_class = formatter,
description="""
Create a new empty stream at the
specified path and with the specifed
layout type.
""")
cmd.set_defaults(handler = self.cmd_create)
group = cmd.add_argument_group("Required arguments")
group.add_argument("path",
help="Path of new stream, e.g. /foo/bar")
group.add_argument("type",
help="Layout type for new stream, e.g. RawData")
# parser.add_argument_group(group)
# metadata
cmd = sub.add_parser("metadata", help="Get or set stream metadata",
description="""
Get or set key=value metadata associated with
a stream.
""",
usage="%(prog)s path [-g [key ...] | "
"-s key=value [...] | -u key=value [...]]")
cmd.set_defaults(handler = self.cmd_metadata)
# group = OptionGroup(parser, "Stream Operations")
# group.add_argument("-l", "--list", action="store_true", default=False,
# action="store", dest="url",
# default="http://localhost:12380/",
# help="NilmDB server URL (default: %default)")
# parser.add_argument_group(group)
group = cmd.add_argument_group("Required arguments")
group.add_argument("path",
help="Path of stream, e.g. /foo/bar")
group = cmd.add_argument_group("Actions")
exc = group.add_mutually_exclusive_group()
exc.add_argument("-g", "--get", nargs="*", metavar="key",
help="Get metadata for specified keys (default all)")
exc.add_argument("-s", "--set", nargs="+", metavar="key=value",
help="Replace all metadata with provided "
"key=value pairs")
exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
help="Update metadata using provided "
"key=value pairs")
# parse it
self.args = parser.parse_args(self.argv)
def die(self, formatstr, *args):
@ -97,13 +129,62 @@ class Cmdline(object):
# Now dispatch client request to appropriate function. Parser
# should have ensured that we don't have any unknown commands
# here.
getattr(self,"cmd_" + self.args.command)()
self.args.handler()
def cmd_info(self):
"""Print info about the server"""
printf("Client library version: %s\n", self.client.client_version)
printf("Server version: %s\n", self.client.version())
printf("Server URL: %s\n", self.client.geturl())
printf("Server database: %s\n", self.client.dbpath())
# if not opt.quiet:
# printf("Server URL: %s\n", opt.url)
def cmd_list(self):
"""List available streams"""
streams = self.client.stream_list()
for (path, type) in streams:
if (fnmatch.fnmatch(path, self.args.path) and
fnmatch.fnmatch(type, self.args.type)):
printf("%s %s\n", path, type)
def cmd_create(self):
"""Create new stream"""
try:
self.client.stream_create(self.args.path, self.args.type)
except nilmdb.client.ClientError as e:
self.die("Error creating stream: %s\n", str(e))
def cmd_metadata(self):
"""Manipulate metadata"""
if self.args.set is not None or self.args.update is not None:
# Either a set or an update
if self.args.set is not None:
keyvals = self.args.set
handler = self.client.stream_set_metadata
else:
keyvals = self.args.update
handler = self.client.stream_update_metadata
# Extract key=value pairs
data = {}
for keyval in keyvals:
kv = keyval.split('=', 1)
if len(kv) != 2 or kv[0] == "":
self.die("Error parsing key=value argument '%s'\n", keyval)
data[kv[0]] = kv[1]
# Make the call
try:
handler(self.args.path, data)
except nilmdb.client.ClientError as e:
self.die("Error setting/updating metadata: %s\n", str(e))
else:
# Get (or unspecified)
keys = self.args.get or None
try:
data = self.client.stream_get_metadata(self.args.path, keys)
except nilmdb.client.ClientError as e:
self.die("Error getting metadata: %s\n", str(e))
for key, value in sorted(data.items()):
if value is None:
value = ""
printf("%s=%s\n", key, value)

14
nilmdb/nilmdb.py
View File

@ -196,7 +196,12 @@ class NilmDB(object):
def stream_create(self, path, layout_name, index = None):
"""Create a new table in the database.
path: path to the data (e.g. '/newton/prep')
path: path to the data (e.g. '/newton/prep').
Paths must contain at least two elements, e.g.:
/newton/prep
/newton/raw
/newton/upstairs/prep
/newton/upstairs/raw
layout_name: one of the nilmdb.layout.layouts keys, e.g. 'PrepData'
@ -208,7 +213,7 @@ class NilmDB(object):
raise ValueError("paths must start with /")
[ group, node ] = path.rsplit("/", 1)
if group == '':
raise ValueError("Invalid path")
raise ValueError("invalid path")
# Make the group structure, one element at a time
group_path = group.lstrip('/').split("/")
@ -221,7 +226,10 @@ class NilmDB(object):
pass
# Get description
desc = nilmdb.layout.named[layout_name].description()
try:
desc = nilmdb.layout.named[layout_name].description()
except KeyError:
raise ValueError("no such layout")
# Estimated table size (for PyTables optimization purposes): assume
# 3 months worth of data. It's OK if this is wrong.

2
nilmtool.py
View File

@ -3,4 +3,4 @@
import nilmdb
import sys
nilmdb.cmdline.run(sys.argv[1:])
nilmdb.cmdline.Cmdline(sys.argv[1:]).run()

6
tests/test_client.py
View File

@ -114,6 +114,12 @@ class TestClient(object):
eq_(client.stream_get_metadata("/newton/raw", [ "description",
"v_scale" ] ), meta1)
# missing key
eq_(client.stream_get_metadata("/newton/raw", "descr"),
{ "descr": None })
eq_(client.stream_get_metadata("/newton/raw", [ "descr" ]),
{ "descr": None })
# test wrong types (list instead of dict)
with assert_raises(ClientError):
client.stream_set_metadata("/newton/prep", [1,2,3])

168
tests/test_cmdline.py
View File

@ -41,7 +41,7 @@ def teardown_module():
class TestCmdline(object):
def run(self, arg_string, input_string = ""):
def run(self, arg_string, input_string = "", capture_stderr=True):
"""Run a cmdline client with the specified argument string,
passing the given input. Returns a tuple with the output and
exit code"""
@ -55,7 +55,11 @@ class TestCmdline(object):
( sys.stdin, sys.stdout, sys.stderr ) = self.saved
infile = cStringIO.StringIO(input_string)
outfile = cStringIO.StringIO()
with stdio_wrapper(infile, outfile, outfile) as s:
if capture_stderr:
errfile = outfile
else:
errfile = sys.stderr
with stdio_wrapper(infile, outfile, errfile) as s:
try:
nilmdb.cmdline.Cmdline(shlex.split(arg_string)).run()
sys.exit(0)
@ -80,12 +84,13 @@ class TestCmdline(object):
def check(self, checkstring):
in_(checkstring, self.output)
def dump(self):
print '-----dump start-----'
print self.output[:-1]
print '-----dump end-----'
def match(self, checkstring):
eq_(checkstring, self.output)
def test_cmdline_basic(self):
def dump(self):
printf("-----dump start-----\n%s-----dump end-----\n", self.output)
def test_cmdline_1_basic(self):
# help
self.ok("--help")
@ -116,8 +121,155 @@ class TestCmdline(object):
self.ok("-u localhost:12380 info")
self.ok("info")
def test_cmdline_info(self):
def test_cmdline_2_info(self):
self.ok("info")
self.check("Server URL: http://localhost:12380/")
self.check("Server version: " + test_server.version)
def test_cmdline_3_misc(self):
# Basic stream tests, like those in test_client.
# BUG: for some reason these start to hang up! what's going on?
for i in range(100):
print i
self.ok("list")
eq_(1,0)
# No streams
self.ok("list")
self.match("")
# Bad paths
self.fail("create foo/bar/baz PrepData")
self.check("paths must start with /")
self.fail("create /foo PrepData")
self.check("invalid path")
# Bad layout type
self.fail("create /newton/prep NoSuchLayout")
self.check("no such layout")
# Create a few streams
self.ok("create /newton/prep PrepData")
self.ok("create /newton/raw RawData")
self.ok("create /newton/zzz/rawnotch RawNotchedData")
# Verify we got those 3 streams
self.ok("list")
self.match("/newton/prep PrepData\n"
"/newton/raw RawData\n"
"/newton/zzz/rawnotch RawNotchedData\n")
# Match just one type or one path
self.ok("list --path /newton/raw")
self.match("/newton/raw RawData\n")
self.ok("list --type RawData")
self.match("/newton/raw RawData\n")
# Wildcard matches
self.ok("list --type Raw*")
self.match("/newton/raw RawData\n"
"/newton/zzz/rawnotch RawNotchedData\n")
self.ok("list --path *zzz* --type Raw*")
self.match("/newton/zzz/rawnotch RawNotchedData\n")
self.ok("list --path *zzz* --type Prep*")
self.match("")
# Set / get metadata
self.fail("metadata")
self.fail("metadata --get")
self.ok("metadata /newton/prep")
self.match("")
self.ok("metadata /newton/raw --get")
self.match("")
self.ok("metadata /newton/prep --set "
"'description=The Data' "
"v_scale=1.234")
self.ok("metadata /newton/raw --update "
"'description=The Data'")
self.ok("metadata /newton/raw --update "
"v_scale=1.234")
self.ok("metadata /newton/prep")
self.match("description=The Data\nv_scale=1.234\n")
self.ok("metadata /newton/prep --get")
self.match("description=The Data\nv_scale=1.234\n")
self.ok("metadata /newton/prep --get descr")
self.match("")
self.ok("metadata /newton/prep --get description")
self.match("description=The Data\n")
self.ok("metadata /newton/raw")
self.dump()
# client.stream_set_metadata("/newton/prep", meta1)
# client.stream_update_metadata("/newton/prep", {})
# client.stream_update_metadata("/newton/raw", meta2)
# client.stream_update_metadata("/newton/raw", meta3)
# eq_(client.stream_get_metadata("/newton/prep"), meta1)
# eq_(client.stream_get_metadata("/newton/raw"), meta1)
# eq_(client.stream_get_metadata("/newton/raw", [ "description" ] ), meta2)
# eq_(client.stream_get_metadata("/newton/raw", [ "description",
# "v_scale" ] ), meta1)
# # test wrong types (list instead of dict)
# with assert_raises(ClientError):
# client.stream_set_metadata("/newton/prep", [1,2,3])
# with assert_raises(ClientError):
# client.stream_update_metadata("/newton/prep", [1,2,3])
# def test_client_3_insert(self):
# client = nilmdb.Client(url = "http://localhost:12380/")
# datetime_tz.localtz_set("America/New_York")
# testfile = "tests/data/prep-20120323T1000"
# start = datetime_tz.datetime_tz.smartparse("20120323T1000")
# rate = 120
# # First try a nonexistent path
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# with assert_raises(ClientError) as e:
# result = client.stream_insert("/newton/no-such-path", data)
# in_("404 Not Found", str(e.exception))
# # Now try reversed timestamps
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# data = reversed(list(data))
# with assert_raises(ClientError) as e:
# result = client.stream_insert("/newton/prep", data)
# in_("400 Bad Request", str(e.exception))
# in_("timestamp is not monotonically increasing", str(e.exception))
# # Now try empty data (no server request made)
# empty = cStringIO.StringIO("")
# data = nilmdb.timestamper.TimestamperRate(empty, start, 120)
# result = client.stream_insert("/newton/prep", data)
# eq_(result, None)
# # Try forcing a server request with empty data
# with assert_raises(ClientError) as e:
# client.curl.putjson("stream/insert", "", { "path": "/newton/prep" })
# in_("400 Bad Request", str(e.exception))
# in_("no data provided", str(e.exception))
# # Now do the real load
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# result = client.stream_insert("/newton/prep", data)
# eq_(result, "ok")
# # Try some overlapping data -- just insert it again
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# with assert_raises(ClientError) as e:
# result = client.stream_insert("/newton/prep", data)
# in_("400 Bad Request", str(e.exception))
# in_("OverlapError", str(e.exception))

20
tests/test_helpers.py
View File

@ -2,25 +2,23 @@
import shutil, os
def myrepr(x):
if isinstance(x, basestring):
return '"' + x + '"'
else:
return repr(x)
def eq_(a, b):
if not a == b:
raise AssertionError("%r != %r" % (a, b))
raise AssertionError("%s != %s" % (myrepr(a), myrepr(b)))
def in_(a, b):
if a not in b:
if not isinstance(a, basestring):
a = repr(a)
else:
a = '"' + a + '"'
if not isinstance(b, basestring):
b = repr(b)
else:
b = '"' + b + '"'
raise AssertionError("%s not in %s" % (a, b))
raise AssertionError("%s not in %s" % (myrepr(a), myrepr(b)))
def ne_(a, b):
if not a != b:
raise AssertionError("unexpected %r == %r" % (a, b))
raise AssertionError("unexpected %s == %s" % (myrepr(a), myrepr(b)))
def recursive_unlink(path):
try: