Browse Source

Support deleting streams with new 'destroy' command

tags/before-insert-rework
Jim Paris 9 years ago
parent
commit
5fecec2a4c
8 changed files with 144 additions and 12 deletions
  1. +1
    -0
      .gitignore
  2. +5
    -0
      nilmdb/client.py
  3. +2
    -1
      nilmdb/cmdline/cmdline.py
  4. +25
    -0
      nilmdb/cmdline/destroy.py
  5. +38
    -7
      nilmdb/nilmdb.py
  6. +11
    -0
      nilmdb/server.py
  7. +59
    -2
      tests/test_cmdline.py
  8. +3
    -2
      timeit.sh

+ 1
- 0
.gitignore View File

@@ -1,3 +1,4 @@
db/
tests/*testdb/
.coverage
*.pyc

+ 5
- 0
nilmdb/client.py View File

@@ -84,6 +84,11 @@ class Client(object):
"layout" : layout }
return self.http.get("stream/create", params)

def stream_destroy(self, path):
"""Delete stream and its contents"""
params = { "path": path }
return self.http.get("stream/destroy", params)

def stream_insert(self, path, data):
"""Insert data into a stream. data should be a file-like object
that provides ASCII data that matches the database layout for path."""


+ 2
- 1
nilmdb/cmdline/cmdline.py View File

@@ -15,7 +15,8 @@ version = "0.1"

# Valid subcommands. Defined in separate files just to break
# things up -- they're still called with Cmdline as self.
subcommands = [ "info", "create", "list", "metadata", "insert", "extract" ]
subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
"destroy" ]

# Import the subcommand modules. Equivalent way of doing this would be
# from . import info as cmd_info


+ 25
- 0
nilmdb/cmdline/destroy.py View File

@@ -0,0 +1,25 @@
from __future__ import absolute_import
from nilmdb.printf import *
import nilmdb.client

from argparse import ArgumentDefaultsHelpFormatter as def_form

def setup(self, sub):
cmd = sub.add_parser("destroy", help="Delete a stream and all data",
formatter_class = def_form,
description="""
Destroy the stream at the specified path. All
data and metadata related to the stream is
permanently deleted.
""")
cmd.set_defaults(handler = cmd_destroy)
group = cmd.add_argument_group("Required arguments")
group.add_argument("path",
help="Path of the stream to delete, e.g. /foo/bar")

def cmd_destroy(self):
"""Destroy stream"""
try:
self.client.stream_destroy(self.args.path)
except nilmdb.client.ClientError as e:
self.die("Error deleting stream: %s", str(e))

+ 38
- 7
nilmdb/nilmdb.py View File

@@ -302,10 +302,15 @@ class NilmDB(object):
exp_rows = 8000 * 60*60*24*30*3

# Create the table
table = self.h5file.createTable(group, node,
description = desc,
expectedrows = exp_rows,
createparents = True)
try:
table = self.h5file.createTable(group, node,
description = desc,
expectedrows = exp_rows,
createparents = True)
except AttributeError:
# Trying to create e.g. /foo/bar/baz when /foo/bar is already
# a table raises this error.
raise ValueError("error creating table at that path")

# Insert into SQL database once the PyTables is happy
with self.con as con:
@@ -328,8 +333,7 @@ class NilmDB(object):
"""
stream_id = self._stream_id(path)
with self.con as con:
con.execute("DELETE FROM metadata "
"WHERE stream_id=?", (stream_id,))
con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
for key in data:
if data[key] != '':
con.execute("INSERT INTO metadata VALUES (?, ?, ?)",
@@ -352,6 +356,33 @@ class NilmDB(object):
data.update(newdata)
self.stream_set_metadata(path, data)

def stream_destroy(self, path):
"""Fully remove a table and all of its data from the database.
No way to undo it! The group structure is removed, if there
are no other tables in it. Metadata is removed."""
stream_id = self._stream_id(path)

# Delete the cached interval data
if stream_id in self._cached_iset:
del self._cached_iset[stream_id]

# Delete the data node, and all parent nodes (if they have no
# remaining children)
split_path = path.lstrip('/').split("/")
while split_path:
name = split_path.pop()
where = "/" + "/".join(split_path)
try:
self.h5file.removeNode(where, name, recursive = False)
except tables.NodeError:
break

# Delete metadata, stream, intervals
with self.con as con:
con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
con.execute("DELETE FROM streams WHERE id=?", (stream_id,))

def stream_insert(self, path, parser, old_timestamp = None):
"""Insert new data into the database.
path: Path at which to add the data
@@ -377,7 +408,7 @@ class NilmDB(object):
iset = self._get_intervals(stream_id)
interval = Interval(min_timestamp, parser.max_timestamp)
if iset.intersects(interval):
raise OverlapError("new data overlaps existing data: "
raise OverlapError("new data overlaps existing data at range: "
+ str(iset & interval))

# Insert the data into pytables


+ 11
- 0
nilmdb/server.py View File

@@ -88,6 +88,17 @@ class Stream(NilmApp):
message = sprintf("%s: %s", type(e).__name__, e.message)
raise cherrypy.HTTPError("400 Bad Request", message)

# /stream/destroy?path=/newton/prep
@cherrypy.expose
@cherrypy.tools.json_out()
def destroy(self, path):
"""Delete a stream and its associated data."""
try:
return self.db.stream_destroy(path)
except Exception as e:
message = sprintf("%s: %s", type(e).__name__, e.message)
raise cherrypy.HTTPError("400 Bad Request", message)

# /stream/get_metadata?path=/newton/prep
# /stream/get_metadata?path=/newton/prep&key=foo&key=bar
@cherrypy.expose


+ 59
- 2
tests/test_cmdline.py View File

@@ -192,11 +192,17 @@ class TestCmdline(object):
self.contain("no such layout")

# Create a few streams
self.ok("create /newton/zzz/rawnotch RawNotchedData")
self.ok("create /newton/prep PrepData")
self.ok("create /newton/raw RawData")
self.ok("create /newton/zzz/rawnotch RawNotchedData")

# Verify we got those 3 streams
# Should not be able to create a stream with another stream as
# its parent
self.fail("create /newton/prep/blah PrepData")
self.contain("error creating table at that path")

# Verify we got those 3 streams and they're returned in
# alphabetical order.
self.ok("list")
self.match("/newton/prep PrepData\n"
"/newton/raw RawData\n"
@@ -456,3 +462,54 @@ class TestCmdline(object):
eq_(self.captured.count('\n'), 11)
server_stop()
server_start()

def test_cmdline_10_destroy(self):
# Delete records
self.ok("destroy --help")

self.fail("destroy")
self.contain("too few arguments")

self.fail("destroy /no/such/stream")
self.contain("No stream at path")

self.fail("destroy asdfasdf")
self.contain("No stream at path")

# From previous tests, we have:
self.ok("list")
self.match("/newton/prep PrepData\n"
"/newton/raw RawData\n"
"/newton/zzz/rawnotch RawNotchedData\n")

# Notice how they're not empty
self.ok("list --detail")
eq_(self.captured.count('\n'), 11)

# Delete some
self.ok("destroy /newton/prep")
self.ok("list")
self.match("/newton/raw RawData\n"
"/newton/zzz/rawnotch RawNotchedData\n")

self.ok("destroy /newton/zzz/rawnotch")
self.ok("list")
self.match("/newton/raw RawData\n")

self.ok("destroy /newton/raw")
self.ok("create /newton/raw RawData")
self.ok("destroy /newton/raw")
self.ok("list")
self.match("")

# Re-create a previously deleted location, and some new ones
rebuild = [ "/newton/prep", "/newton/zzz",
"/newton/raw", "/newton/asdf/qwer" ]
for path in rebuild:
# Create the path
self.ok("create " + path + " PrepData")
self.ok("list")
self.contain(path)
# Make sure it was created empty
self.ok("list --detail --path " + path)
self.contain("(no intervals)")

+ 3
- 2
timeit.sh View File

@@ -1,8 +1,9 @@
./nilmtool.py destroy /bpnilm/2/raw
./nilmtool.py create /bpnilm/2/raw RawData

if true; then
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/2/raw
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/2/raw
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 -r 8000 /bpnilm/2/raw
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 -r 8000 /bpnilm/2/raw
else
for i in $(seq 2000 2050); do
time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-010001 /bpnilm/2/raw


Loading…
Cancel
Save