Browse Source

Lots of updates to generic filter, specific decimate module

tags/nilmtools-0.2
Jim Paris 9 years ago
parent
commit
4243301434
4 changed files with 146 additions and 28 deletions
  1. +3
    -0
      Makefile
  2. +59
    -0
      nilmtools/decimate.py
  3. +74
    -24
      nilmtools/filter.py
  4. +10
    -4
      nilmtools/insert.py

+ 3
- 0
Makefile View File

@@ -1,3 +1,6 @@
test:
python nilmtools/decimate.py /lees-compressor/noleak/raw /lees-compressor/noleak/raw~4

all:
@echo "Try 'make install'"



+ 59
- 0
nilmtools/decimate.py View File

@@ -0,0 +1,59 @@
#!/usr/bin/python

import nilmtools.filter
import nilmdb.client

def DecimateException(Exception):
pass

def main():
f = nilmtools.filter.Filter()
parser = f.setup_parser("Decimate a stream")
group = parser.add_argument_group("Decimate options")
group.add_argument('-f', '--factor', action='store', default=4, type=int,
help='Decimation factor (default: %(default)s)')
# Parse arguments
try:
args = f.parse_args()
except nilmtools.filter.MissingDestination as e:
# If no destination, suggest how to create it.
print "Source is %s (%s)" % (e.src, e.layout)
print "Destination %s doesn't exist" % (e.dest)
# Figure out a recommended layout
if "decimate_source" in f.client.stream_get_metadata(e.src):
rec = e.layout
elif 'int32' in e.layout_type or 'float64' in e.layout_type:
rec = 'float64_' + str(e.layout_count * 3)
else:
rec = 'float32_' + str(e.layout_count * 3)
print "You could make it with a command like:"
print " nilmtool create", e.dest, rec
raise SystemExit(1)

# See if the metadata jives, and complain if it doesn't
dest_metadata = f.client.stream_get_metadata(args.destpath)
try:
rows = f.destinfo[4] # don't complain unless there's data
tmp = dest_metadata.get("decimate_source", args.srcpath)
if tmp != args.srcpath and rows > 0:
raise DecimateException("storing decimated data from %s" % tmp)
tmp = int(dest_metadata.get("decimate_factor", args.factor))
if tmp != args.factor and rows > 0:
raise DecimateException("storing data decimated at "
"a different factor (%d)", tmp)
except DecimateException as e:
print "The destination seems to already be " + str(e)
print "Refusing to change it. You can change the stream's"
print "decimate_* metadata, or remove all data in the stream,"
print "to prevent this error."

# Fill in the metadata in case it's missing
f.client.stream_update_metadata(args.destpath,
{ "decimate_source": args.srcpath,
"decimate_factor": args.factor })

# Process it

if __name__ == "__main__":
main()

+ 74
- 24
nilmtools/filter.py View File

@@ -4,63 +4,113 @@ import nilmdb.client
from nilmdb.utils.printf import *
from nilmdb.utils.time import parse_time, format_time

import nilmtools
import time
import sys
import re
import argparse

class MissingDestination(Exception):
def __init__(self, src, layout, dest):
self.src = src
self.layout = layout
self.layout_type = layout.split('_')[0]
self.layout_count = int(layout.split('_')[1])
self.dest = dest
Exception.__init__(self, "destination path " + dest + " not found")

class Filter(object):

def __init__(self, description = "Filter data"):
self.args = None
def __init__(self):
self._parser = None
self._args = None
self._client = None
self.parse_args(description)
self._using_client = False
self.srcinfo = None
self.destinfo = None

@property
def client(self):
if self._using_client:
raise Exception("Filter client is in use; make another")
return self._client

def parse_args(self, description):
def setup_parser(self, description = "Filter data"):
parser = argparse.ArgumentParser(
description = description,
formatter_class = argparse.RawDescriptionHelpFormatter)
parser.add_argument("-u", "--url", action="store",
default="http://localhost:12380/",
help="Server URL (default: %(default)s)")
parser.add_argument("srcpath", action="store",
help="Path of source stream, e.g. /foo/bar")
parser.add_argument("destpath", action="store",
help="Path of destination stream, e.g. /foo/bar")
self.args = parser.parse_args()
formatter_class = argparse.RawDescriptionHelpFormatter,
version = nilmtools.__version__,
description = description)
group = parser.add_argument_group("General filter arguments")
group.add_argument("-u", "--url", action="store",
default="http://localhost:12380/",
help="Server URL (default: %(default)s)")
group.add_argument("-D", "--dry-run", action="store_true",
default = False,
help="Just print intervals that would be "
"processed")
group.add_argument("srcpath", action="store",
help="Path of source stream, e.g. /foo/bar")
group.add_argument("destpath", action="store",
help="Path of destination stream, e.g. /foo/bar")
self._parser = parser
return parser

def parse_args(self):
args = self._parser.parse_args()
self._args = args
self._client = nilmdb.client.Client(args.url)

if args.srcpath == args.destpath:
raise Exception("source and destination path must be different")

# Open and print info about the streams
def stream_info_string(info):
return sprintf("%s (%s), %.2fM rows, %.2f hours\n",
info[0], info[1], info[4] / 1e6, info[5] / 3600)

src = self._client.stream_list(args.srcpath, extended = True)
if len(src) != 1:
raise Exception("source path " + args.srcpath + " not found")
print "Source:", stream_info_string(src[0])
self.srcinfo = src[0]

dest = self._client.stream_list(args.destpath, extended = True)
if len(dest) != 1:
raise Exception("destination path " + args.destpath + " not found")
print " Dest:", stream_info_string(dest[0])
raise MissingDestination(self.srcinfo[0], self.srcinfo[1],
args.destpath)
self.destinfo = dest[0]

print "Source:", self.stream_info_string(self.srcinfo)
print " Dest:", self.stream_info_string(self.destinfo)

if args.dry_run:
for interval in self.intervals():
print self.interval_string(interval)
raise SystemExit(0)

return args

def intervals(self):
"""Generate all the intervals that this filter should process"""
self._using_client = True
for i in self._client.stream_intervals(
args.srcpath, diffpath = args.destpath):
self._args.srcpath, diffpath = self._args.destpath):
yield i
self._using_client = False

# Misc helpers
def stream_info_string(self, info):
return sprintf("%s (%s), %.2fM rows, %.2f hours",
info[0], info[1], info[4] / 1e6, info[5] / 3600)

def interval_string(self, interval):
return sprintf("[ %s -> %s ]", format_time(interval[0]),
format_time(interval[1]))


def main():
# This is just a dummy function; actual filters can use the other
# functions to prepare stuff, and then do something with the data.
f = Filter()
for interval in f.intervals():
print "Generic filter: need to handle interval:", interval
parser = f.setup_parser()
args = f.parse_args()
for (start, end) in f.intervals():
print "Generic filter: need to handle", start, " to ", end

if __name__ == "__main__":
main()

+ 10
- 4
nilmtools/insert.py View File

@@ -4,6 +4,7 @@ import nilmdb.client
from nilmdb.utils.printf import *
from nilmdb.utils.time import parse_time, format_time

import nilmtools
import time
import sys
import re
@@ -16,7 +17,10 @@ class ParseError(Exception):
super(ParseError, self).__init__(msg)

def parse_args():
parser = argparse.ArgumentParser(description = """\
parser = argparse.ArgumentParser(
formatter_class = argparse.RawDescriptionHelpFormatter,
version = nilmtools.__version__,
description = """\
Insert data from ethstream, either live (using the system time as a
reference) or prerecorded (using comments in the file as a reference).

@@ -24,18 +28,20 @@ def parse_args():
Small discrepencies between the accumulated timestamps and the
reference time are ignored; larger discrepencies cause gaps to be
created in the stream. Overlapping data returns an error.
""", formatter_class = argparse.RawDescriptionHelpFormatter)
""")
parser.add_argument("-u", "--url", action="store",
default="http://localhost:12380/",
help="NilmDB server URL (default: %(default)s)")
parser.add_argument("-r", "--rate", action="store", default=8000, type=float,
parser.add_argument("-r", "--rate", action="store", default=8000,
type=float,
help="Data rate in Hz (default: %(default)s)")
parser.add_argument("-l", "--live", action="store_true",
help="Live capture; use system time to verify rate")
parser.add_argument("path", action="store",
help="Path of stream, e.g. /foo/bar")
parser.add_argument("infile", type=argparse.FileType('r'), nargs='*',
default=[sys.stdin], help="Input files (default: stdin)")
default=[sys.stdin],
help="Input files (default: stdin)")
args = parser.parse_args()

printf("Stream path: %s\n", args.path)


Loading…
Cancel
Save