@@ -1,3 +1,4 @@ | |||
db/ | |||
tests/*testdb/ | |||
.coverage | |||
*.pyc |
@@ -84,6 +84,11 @@ class Client(object): | |||
"layout" : layout } | |||
return self.http.get("stream/create", params) | |||
def stream_destroy(self, path): | |||
"""Delete stream and its contents""" | |||
params = { "path": path } | |||
return self.http.get("stream/destroy", params) | |||
def stream_insert(self, path, data): | |||
"""Insert data into a stream. data should be a file-like object | |||
that provides ASCII data that matches the database layout for path.""" | |||
@@ -15,7 +15,8 @@ version = "0.1" | |||
# Valid subcommands. Defined in separate files just to break | |||
# things up -- they're still called with Cmdline as self. | |||
subcommands = [ "info", "create", "list", "metadata", "insert", "extract" ] | |||
subcommands = [ "info", "create", "list", "metadata", "insert", "extract", | |||
"destroy" ] | |||
# Import the subcommand modules. Equivalent way of doing this would be | |||
# from . import info as cmd_info | |||
@@ -0,0 +1,25 @@ | |||
from __future__ import absolute_import | |||
from nilmdb.printf import * | |||
import nilmdb.client | |||
from argparse import ArgumentDefaultsHelpFormatter as def_form | |||
def setup(self, sub): | |||
cmd = sub.add_parser("destroy", help="Delete a stream and all data", | |||
formatter_class = def_form, | |||
description=""" | |||
Destroy the stream at the specified path. All | |||
data and metadata related to the stream is | |||
permanently deleted. | |||
""") | |||
cmd.set_defaults(handler = cmd_destroy) | |||
group = cmd.add_argument_group("Required arguments") | |||
group.add_argument("path", | |||
help="Path of the stream to delete, e.g. /foo/bar") | |||
def cmd_destroy(self): | |||
"""Destroy stream""" | |||
try: | |||
self.client.stream_destroy(self.args.path) | |||
except nilmdb.client.ClientError as e: | |||
self.die("Error deleting stream: %s", str(e)) |
@@ -302,10 +302,15 @@ class NilmDB(object): | |||
exp_rows = 8000 * 60*60*24*30*3 | |||
# Create the table | |||
table = self.h5file.createTable(group, node, | |||
description = desc, | |||
expectedrows = exp_rows, | |||
createparents = True) | |||
try: | |||
table = self.h5file.createTable(group, node, | |||
description = desc, | |||
expectedrows = exp_rows, | |||
createparents = True) | |||
except AttributeError: | |||
# Trying to create e.g. /foo/bar/baz when /foo/bar is already | |||
# a table raises this error. | |||
raise ValueError("error creating table at that path") | |||
# Insert into SQL database once the PyTables is happy | |||
with self.con as con: | |||
@@ -328,8 +333,7 @@ class NilmDB(object): | |||
""" | |||
stream_id = self._stream_id(path) | |||
with self.con as con: | |||
con.execute("DELETE FROM metadata " | |||
"WHERE stream_id=?", (stream_id,)) | |||
con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,)) | |||
for key in data: | |||
if data[key] != '': | |||
con.execute("INSERT INTO metadata VALUES (?, ?, ?)", | |||
@@ -352,6 +356,33 @@ class NilmDB(object): | |||
data.update(newdata) | |||
self.stream_set_metadata(path, data) | |||
def stream_destroy(self, path): | |||
"""Fully remove a table and all of its data from the database. | |||
No way to undo it! The group structure is removed, if there | |||
are no other tables in it. Metadata is removed.""" | |||
stream_id = self._stream_id(path) | |||
# Delete the cached interval data | |||
if stream_id in self._cached_iset: | |||
del self._cached_iset[stream_id] | |||
# Delete the data node, and all parent nodes (if they have no | |||
# remaining children) | |||
split_path = path.lstrip('/').split("/") | |||
while split_path: | |||
name = split_path.pop() | |||
where = "/" + "/".join(split_path) | |||
try: | |||
self.h5file.removeNode(where, name, recursive = False) | |||
except tables.NodeError: | |||
break | |||
# Delete metadata, stream, intervals | |||
with self.con as con: | |||
con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,)) | |||
con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,)) | |||
con.execute("DELETE FROM streams WHERE id=?", (stream_id,)) | |||
def stream_insert(self, path, parser, old_timestamp = None): | |||
"""Insert new data into the database. | |||
path: Path at which to add the data | |||
@@ -377,7 +408,7 @@ class NilmDB(object): | |||
iset = self._get_intervals(stream_id) | |||
interval = Interval(min_timestamp, parser.max_timestamp) | |||
if iset.intersects(interval): | |||
raise OverlapError("new data overlaps existing data: " | |||
raise OverlapError("new data overlaps existing data at range: " | |||
+ str(iset & interval)) | |||
# Insert the data into pytables | |||
@@ -88,6 +88,17 @@ class Stream(NilmApp): | |||
message = sprintf("%s: %s", type(e).__name__, e.message) | |||
raise cherrypy.HTTPError("400 Bad Request", message) | |||
# /stream/destroy?path=/newton/prep | |||
@cherrypy.expose | |||
@cherrypy.tools.json_out() | |||
def destroy(self, path): | |||
"""Delete a stream and its associated data.""" | |||
try: | |||
return self.db.stream_destroy(path) | |||
except Exception as e: | |||
message = sprintf("%s: %s", type(e).__name__, e.message) | |||
raise cherrypy.HTTPError("400 Bad Request", message) | |||
# /stream/get_metadata?path=/newton/prep | |||
# /stream/get_metadata?path=/newton/prep&key=foo&key=bar | |||
@cherrypy.expose | |||
@@ -192,11 +192,17 @@ class TestCmdline(object): | |||
self.contain("no such layout") | |||
# Create a few streams | |||
self.ok("create /newton/zzz/rawnotch RawNotchedData") | |||
self.ok("create /newton/prep PrepData") | |||
self.ok("create /newton/raw RawData") | |||
self.ok("create /newton/zzz/rawnotch RawNotchedData") | |||
# Verify we got those 3 streams | |||
# Should not be able to create a stream with another stream as | |||
# its parent | |||
self.fail("create /newton/prep/blah PrepData") | |||
self.contain("error creating table at that path") | |||
# Verify we got those 3 streams and they're returned in | |||
# alphabetical order. | |||
self.ok("list") | |||
self.match("/newton/prep PrepData\n" | |||
"/newton/raw RawData\n" | |||
@@ -456,3 +462,54 @@ class TestCmdline(object): | |||
eq_(self.captured.count('\n'), 11) | |||
server_stop() | |||
server_start() | |||
def test_cmdline_10_destroy(self): | |||
# Delete records | |||
self.ok("destroy --help") | |||
self.fail("destroy") | |||
self.contain("too few arguments") | |||
self.fail("destroy /no/such/stream") | |||
self.contain("No stream at path") | |||
self.fail("destroy asdfasdf") | |||
self.contain("No stream at path") | |||
# From previous tests, we have: | |||
self.ok("list") | |||
self.match("/newton/prep PrepData\n" | |||
"/newton/raw RawData\n" | |||
"/newton/zzz/rawnotch RawNotchedData\n") | |||
# Notice how they're not empty | |||
self.ok("list --detail") | |||
eq_(self.captured.count('\n'), 11) | |||
# Delete some | |||
self.ok("destroy /newton/prep") | |||
self.ok("list") | |||
self.match("/newton/raw RawData\n" | |||
"/newton/zzz/rawnotch RawNotchedData\n") | |||
self.ok("destroy /newton/zzz/rawnotch") | |||
self.ok("list") | |||
self.match("/newton/raw RawData\n") | |||
self.ok("destroy /newton/raw") | |||
self.ok("create /newton/raw RawData") | |||
self.ok("destroy /newton/raw") | |||
self.ok("list") | |||
self.match("") | |||
# Re-create a previously deleted location, and some new ones | |||
rebuild = [ "/newton/prep", "/newton/zzz", | |||
"/newton/raw", "/newton/asdf/qwer" ] | |||
for path in rebuild: | |||
# Create the path | |||
self.ok("create " + path + " PrepData") | |||
self.ok("list") | |||
self.contain(path) | |||
# Make sure it was created empty | |||
self.ok("list --detail --path " + path) | |||
self.contain("(no intervals)") |
@@ -1,8 +1,9 @@ | |||
./nilmtool.py destroy /bpnilm/2/raw | |||
./nilmtool.py create /bpnilm/2/raw RawData | |||
if true; then | |||
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/2/raw | |||
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/2/raw | |||
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 -r 8000 /bpnilm/2/raw | |||
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 -r 8000 /bpnilm/2/raw | |||
else | |||
for i in $(seq 2000 2050); do | |||
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-010001 /bpnilm/2/raw | |||