nilm
/
nilmdb
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 64 Wiki Activity
Browse Source

A lot more command line testing. 

There'es some issue with tons of requests getting slowly blocked,
though...



git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10678 ddd99763-3ecb-0310-9145-efcb8ce7c51f
tags/bxinterval-last
Jim Paris 12 years ago
parent
277b0c1d00
commit
a3f444eb25
6 changed files with 286 additions and 41 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +100
    -19
      nilmdb/cmdline.py
  2. +11
    -3
      nilmdb/nilmdb.py
  3. +1
    -1
      nilmtool.py
  4. +6
    -0
      tests/test_client.py
  5. +159
    -7
      tests/test_cmdline.py
  6. +9
    -11
      tests/test_helpers.py

+ 100
- 19
nilmdb/cmdline.py View File

@@ -11,6 +11,7 @@ import re
import os
import urlparse
import argparse
import fnmatch

version = "0.1"

@@ -32,8 +33,6 @@ class Cmdline(object):
help='show this help message and exit')
group.add_argument("-V", "--version", action="version",
version=version_string)
group.add_argument("-q", "--quiet", action="store_true",
help="suppress unnecessary console output")

group = parser.add_argument_group("Server")
group.add_argument("-u", "--url", action="store",
@@ -52,31 +51,64 @@ class Cmdline(object):
List information about the server, like
version.
""")
cmd.set_defaults(handler = self.cmd_info)

# list
cmd = sub.add_parser("list", help="List streams",
formatter_class = formatter,
description="""
List streams available in the database,
optionally filtering by type or partial path.
optionally filtering by type or path. Wildcards
are accepted.
""")
cmd.set_defaults(handler = self.cmd_list)
group = cmd.add_argument_group("Stream filtering")
group.add_argument("-t", "--type", metavar="GLOB", default="*",
group.add_argument("-t", "--type", default="*",
help="Match only this stream type")
group.add_argument("-p", "--path", metavar="GLOB", default="*",
group.add_argument("-p", "--path", default="*",
help="Match only this path")

# group.add_argument(

# parser.add_argument_group(group)

# group = OptionGroup(parser, "Stream Operations")
# group.add_argument("-l", "--list", action="store_true", default=False,
# action="store", dest="url",
# default="http://localhost:12380/",
# help="NilmDB server URL (default: %default)")
# parser.add_argument_group(group)

# create
cmd = sub.add_parser("create", help="Create a new stream",
formatter_class = formatter,
description="""
Create a new empty stream at the
specified path and with the specifed
layout type.
""")
cmd.set_defaults(handler = self.cmd_create)
group = cmd.add_argument_group("Required arguments")
group.add_argument("path",
help="Path of new stream, e.g. /foo/bar")
group.add_argument("type",
help="Layout type for new stream, e.g. RawData")

# metadata
cmd = sub.add_parser("metadata", help="Get or set stream metadata",
description="""
Get or set key=value metadata associated with
a stream.
""",
usage="%(prog)s path [-g [key ...] | "
"-s key=value [...] | -u key=value [...]]")
cmd.set_defaults(handler = self.cmd_metadata)

group = cmd.add_argument_group("Required arguments")
group.add_argument("path",
help="Path of stream, e.g. /foo/bar")

group = cmd.add_argument_group("Actions")
exc = group.add_mutually_exclusive_group()
exc.add_argument("-g", "--get", nargs="*", metavar="key",
help="Get metadata for specified keys (default all)")
exc.add_argument("-s", "--set", nargs="+", metavar="key=value",
help="Replace all metadata with provided "
"key=value pairs")
exc.add_argument("-u", "--update", nargs="+", metavar="key=value",
help="Update metadata using provided "
"key=value pairs")

# parse it
self.args = parser.parse_args(self.argv)

def die(self, formatstr, *args):
@@ -97,13 +129,62 @@ class Cmdline(object):
# Now dispatch client request to appropriate function. Parser
# should have ensured that we don't have any unknown commands
# here.
getattr(self,"cmd_" + self.args.command)()
self.args.handler()

def cmd_info(self):
"""Print info about the server"""
printf("Client library version: %s\n", self.client.client_version)
printf("Server version: %s\n", self.client.version())
printf("Server URL: %s\n", self.client.geturl())
printf("Server database: %s\n", self.client.dbpath())

# if not opt.quiet:
# printf("Server URL: %s\n", opt.url)
def cmd_list(self):
"""List available streams"""
streams = self.client.stream_list()
for (path, type) in streams:
if (fnmatch.fnmatch(path, self.args.path) and
fnmatch.fnmatch(type, self.args.type)):
printf("%s %s\n", path, type)

def cmd_create(self):
"""Create new stream"""
try:
self.client.stream_create(self.args.path, self.args.type)
except nilmdb.client.ClientError as e:
self.die("Error creating stream: %s\n", str(e))

def cmd_metadata(self):
"""Manipulate metadata"""
if self.args.set is not None or self.args.update is not None:
# Either a set or an update
if self.args.set is not None:
keyvals = self.args.set
handler = self.client.stream_set_metadata
else:
keyvals = self.args.update
handler = self.client.stream_update_metadata

# Extract key=value pairs
data = {}
for keyval in keyvals:
kv = keyval.split('=', 1)
if len(kv) != 2 or kv[0] == "":
self.die("Error parsing key=value argument '%s'\n", keyval)
data[kv[0]] = kv[1]

# Make the call
try:
handler(self.args.path, data)
except nilmdb.client.ClientError as e:
self.die("Error setting/updating metadata: %s\n", str(e))
else:
# Get (or unspecified)
keys = self.args.get or None
try:
data = self.client.stream_get_metadata(self.args.path, keys)
except nilmdb.client.ClientError as e:
self.die("Error getting metadata: %s\n", str(e))
for key, value in sorted(data.items()):
if value is None:
value = ""
printf("%s=%s\n", key, value)

+ 11
- 3
nilmdb/nilmdb.py View File

@@ -196,7 +196,12 @@ class NilmDB(object):
def stream_create(self, path, layout_name, index = None):
"""Create a new table in the database.

path: path to the data (e.g. '/newton/prep')
path: path to the data (e.g. '/newton/prep').
Paths must contain at least two elements, e.g.:
/newton/prep
/newton/raw
/newton/upstairs/prep
/newton/upstairs/raw

layout_name: one of the nilmdb.layout.layouts keys, e.g. 'PrepData'

@@ -208,7 +213,7 @@ class NilmDB(object):
raise ValueError("paths must start with /")
[ group, node ] = path.rsplit("/", 1)
if group == '':
raise ValueError("Invalid path")
raise ValueError("invalid path")

# Make the group structure, one element at a time
group_path = group.lstrip('/').split("/")
@@ -221,7 +226,10 @@ class NilmDB(object):
pass

# Get description
desc = nilmdb.layout.named[layout_name].description()
try:
desc = nilmdb.layout.named[layout_name].description()
except KeyError:
raise ValueError("no such layout")

# Estimated table size (for PyTables optimization purposes): assume
# 3 months worth of data. It's OK if this is wrong.


+ 1
- 1
nilmtool.py View File

@@ -3,4 +3,4 @@
import nilmdb
import sys

nilmdb.cmdline.run(sys.argv[1:])
nilmdb.cmdline.Cmdline(sys.argv[1:]).run()

+ 6
- 0
tests/test_client.py View File

@@ -114,6 +114,12 @@ class TestClient(object):
eq_(client.stream_get_metadata("/newton/raw", [ "description",
"v_scale" ] ), meta1)

# missing key
eq_(client.stream_get_metadata("/newton/raw", "descr"),
{ "descr": None })
eq_(client.stream_get_metadata("/newton/raw", [ "descr" ]),
{ "descr": None })

# test wrong types (list instead of dict)
with assert_raises(ClientError):
client.stream_set_metadata("/newton/prep", [1,2,3])


+ 159
- 7
tests/test_cmdline.py View File

@@ -41,7 +41,7 @@ def teardown_module():

class TestCmdline(object):

def run(self, arg_string, input_string = ""):
def run(self, arg_string, input_string = "", capture_stderr=True):
"""Run a cmdline client with the specified argument string,
passing the given input. Returns a tuple with the output and
exit code"""
@@ -55,7 +55,11 @@ class TestCmdline(object):
( sys.stdin, sys.stdout, sys.stderr ) = self.saved
infile = cStringIO.StringIO(input_string)
outfile = cStringIO.StringIO()
with stdio_wrapper(infile, outfile, outfile) as s:
if capture_stderr:
errfile = outfile
else:
errfile = sys.stderr
with stdio_wrapper(infile, outfile, errfile) as s:
try:
nilmdb.cmdline.Cmdline(shlex.split(arg_string)).run()
sys.exit(0)
@@ -80,12 +84,13 @@ class TestCmdline(object):
def check(self, checkstring):
in_(checkstring, self.output)

def match(self, checkstring):
eq_(checkstring, self.output)

def dump(self):
print '-----dump start-----'
print self.output[:-1]
print '-----dump end-----'
printf("-----dump start-----\n%s-----dump end-----\n", self.output)

def test_cmdline_basic(self):
def test_cmdline_1_basic(self):
# help
self.ok("--help")
@@ -116,8 +121,155 @@ class TestCmdline(object):
self.ok("-u localhost:12380 info")
self.ok("info")

def test_cmdline_info(self):
def test_cmdline_2_info(self):
self.ok("info")
self.check("Server URL: http://localhost:12380/")
self.check("Server version: " + test_server.version)

def test_cmdline_3_misc(self):
# Basic stream tests, like those in test_client.

# BUG: for some reason these start to hang up! what's going on?
for i in range(100):
print i
self.ok("list")
eq_(1,0)

# No streams
self.ok("list")
self.match("")

# Bad paths
self.fail("create foo/bar/baz PrepData")
self.check("paths must start with /")

self.fail("create /foo PrepData")
self.check("invalid path")

# Bad layout type
self.fail("create /newton/prep NoSuchLayout")
self.check("no such layout")

# Create a few streams
self.ok("create /newton/prep PrepData")
self.ok("create /newton/raw RawData")
self.ok("create /newton/zzz/rawnotch RawNotchedData")

# Verify we got those 3 streams
self.ok("list")
self.match("/newton/prep PrepData\n"
"/newton/raw RawData\n"
"/newton/zzz/rawnotch RawNotchedData\n")
# Match just one type or one path
self.ok("list --path /newton/raw")
self.match("/newton/raw RawData\n")

self.ok("list --type RawData")
self.match("/newton/raw RawData\n")

# Wildcard matches
self.ok("list --type Raw*")
self.match("/newton/raw RawData\n"
"/newton/zzz/rawnotch RawNotchedData\n")

self.ok("list --path *zzz* --type Raw*")
self.match("/newton/zzz/rawnotch RawNotchedData\n")

self.ok("list --path *zzz* --type Prep*")
self.match("")

# Set / get metadata
self.fail("metadata")
self.fail("metadata --get")

self.ok("metadata /newton/prep")
self.match("")

self.ok("metadata /newton/raw --get")
self.match("")

self.ok("metadata /newton/prep --set "
"'description=The Data' "
"v_scale=1.234")
self.ok("metadata /newton/raw --update "
"'description=The Data'")
self.ok("metadata /newton/raw --update "
"v_scale=1.234")

self.ok("metadata /newton/prep")
self.match("description=The Data\nv_scale=1.234\n")
self.ok("metadata /newton/prep --get")
self.match("description=The Data\nv_scale=1.234\n")
self.ok("metadata /newton/prep --get descr")
self.match("")
self.ok("metadata /newton/prep --get description")
self.match("description=The Data\n")

self.ok("metadata /newton/raw")
self.dump()


# client.stream_set_metadata("/newton/prep", meta1)
# client.stream_update_metadata("/newton/prep", {})
# client.stream_update_metadata("/newton/raw", meta2)
# client.stream_update_metadata("/newton/raw", meta3)
# eq_(client.stream_get_metadata("/newton/prep"), meta1)
# eq_(client.stream_get_metadata("/newton/raw"), meta1)
# eq_(client.stream_get_metadata("/newton/raw", [ "description" ] ), meta2)
# eq_(client.stream_get_metadata("/newton/raw", [ "description",
# "v_scale" ] ), meta1)

# # test wrong types (list instead of dict)
# with assert_raises(ClientError):
# client.stream_set_metadata("/newton/prep", [1,2,3])
# with assert_raises(ClientError):
# client.stream_update_metadata("/newton/prep", [1,2,3])

# def test_client_3_insert(self):
# client = nilmdb.Client(url = "http://localhost:12380/")

# datetime_tz.localtz_set("America/New_York")

# testfile = "tests/data/prep-20120323T1000"
# start = datetime_tz.datetime_tz.smartparse("20120323T1000")
# rate = 120

# # First try a nonexistent path
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# with assert_raises(ClientError) as e:
# result = client.stream_insert("/newton/no-such-path", data)
# in_("404 Not Found", str(e.exception))

# # Now try reversed timestamps
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# data = reversed(list(data))
# with assert_raises(ClientError) as e:
# result = client.stream_insert("/newton/prep", data)
# in_("400 Bad Request", str(e.exception))
# in_("timestamp is not monotonically increasing", str(e.exception))
# # Now try empty data (no server request made)
# empty = cStringIO.StringIO("")
# data = nilmdb.timestamper.TimestamperRate(empty, start, 120)
# result = client.stream_insert("/newton/prep", data)
# eq_(result, None)

# # Try forcing a server request with empty data
# with assert_raises(ClientError) as e:
# client.curl.putjson("stream/insert", "", { "path": "/newton/prep" })
# in_("400 Bad Request", str(e.exception))
# in_("no data provided", str(e.exception))

# # Now do the real load
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# result = client.stream_insert("/newton/prep", data)
# eq_(result, "ok")

# # Try some overlapping data -- just insert it again
# data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
# with assert_raises(ClientError) as e:
# result = client.stream_insert("/newton/prep", data)
# in_("400 Bad Request", str(e.exception))
# in_("OverlapError", str(e.exception))


+ 9
- 11
tests/test_helpers.py View File

@@ -2,25 +2,23 @@

import shutil, os

def myrepr(x):
if isinstance(x, basestring):
return '"' + x + '"'
else:
return repr(x)

def eq_(a, b):
if not a == b:
raise AssertionError("%r != %r" % (a, b))
raise AssertionError("%s != %s" % (myrepr(a), myrepr(b)))

def in_(a, b):
if a not in b:
if not isinstance(a, basestring):
a = repr(a)
else:
a = '"' + a + '"'
if not isinstance(b, basestring):
b = repr(b)
else:
b = '"' + b + '"'
raise AssertionError("%s not in %s" % (a, b))
raise AssertionError("%s not in %s" % (myrepr(a), myrepr(b)))

def ne_(a, b):
if not a != b:
raise AssertionError("unexpected %r == %r" % (a, b))
raise AssertionError("unexpected %s == %s" % (myrepr(a), myrepr(b)))

def recursive_unlink(path):
try:


Write Preview
Loading…
Cancel
Save