Jim Paris
cfc66b6847
This found a small number of real bugs too, for example, this one that looked weird because of a 2to3 conversion, but was wrong both before and after: - except IndexError as TypeError: + except (IndexError, TypeError):
113 lines
4.0 KiB
Python
Executable File
113 lines
4.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import nilmtools.filter
|
|
import nilmtools.decimate
|
|
import nilmdb.client
|
|
import argparse
|
|
import fnmatch
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
description="""\
|
|
Automatically create multiple decimations from a single source
|
|
stream, continuing until the last decimated level contains fewer
|
|
than 500 points total.
|
|
|
|
Wildcards and multiple paths are accepted. Decimated paths are
|
|
ignored when matching wildcards.
|
|
""")
|
|
def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
|
|
parser.add_argument("-u", "--url", action="store", default=def_url,
|
|
help="NilmDB server URL (default: %(default)s)")
|
|
parser.add_argument("-f", "--factor", action="store", default=4, type=int,
|
|
help='Decimation factor (default: %(default)s)')
|
|
parser.add_argument("-m", "--max", action="store", default=500, type=int,
|
|
help='Maximum number of points in last level ' +
|
|
'(default: %(default)s)')
|
|
parser.add_argument("-F", "--force-metadata", action="store_true",
|
|
default=False,
|
|
help="Force metadata changes if the dest "
|
|
"doesn't match")
|
|
parser.add_argument("-v", "--version", action="version",
|
|
version=nilmtools.__version__)
|
|
parser.add_argument("path", action="store", nargs='+',
|
|
help='Path of base stream')
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.max < 0:
|
|
print("error: bad max, must be nonnegative")
|
|
raise SystemExit(1)
|
|
|
|
# Pull out info about the base stream
|
|
client = nilmdb.client.Client(args.url)
|
|
|
|
# Find list of paths to process
|
|
streams = [str(s[0]) for s in client.stream_list()]
|
|
streams = [s for s in streams if "~decim-" not in s]
|
|
paths = []
|
|
for path in args.path:
|
|
new = fnmatch.filter(streams, str(path))
|
|
if not new:
|
|
print("error: no stream matched path:", path)
|
|
raise SystemExit(1)
|
|
paths.extend(new)
|
|
|
|
for path in paths:
|
|
do_decimation(client, args, path)
|
|
|
|
|
|
def do_decimation(client, args, path):
|
|
print("Decimating", path)
|
|
info = nilmtools.filter.get_stream_info(client, path)
|
|
if not info: # pragma: no cover (only good paths passed above)
|
|
raise Exception("path " + path + " not found")
|
|
|
|
meta = client.stream_get_metadata(path)
|
|
if "decimate_source" in meta:
|
|
print("Stream", path, "was decimated from", meta["decimate_source"])
|
|
print("You need to pass the base stream instead")
|
|
raise SystemExit(1)
|
|
|
|
# Figure out the type we should use for decimated streams
|
|
if ('int32' in info.layout_type or
|
|
'int64' in info.layout_type or
|
|
'float64' in info.layout_type):
|
|
decimated_type = 'float64_' + str(info.layout_count * 3)
|
|
else:
|
|
decimated_type = 'float32_' + str(info.layout_count * 3)
|
|
|
|
# Now do the decimations until we have few enough points
|
|
factor = 1
|
|
while True:
|
|
print("Level", factor, "decimation has", info.rows, "rows")
|
|
if info.rows <= args.max:
|
|
break
|
|
factor *= args.factor
|
|
new_path = "%s~decim-%d" % (path, factor)
|
|
|
|
# Create the stream if needed
|
|
new_info = nilmtools.filter.get_stream_info(client, new_path)
|
|
if not new_info:
|
|
print("Creating stream", new_path)
|
|
client.stream_create(new_path, decimated_type)
|
|
|
|
# Run the decimation as if it were run from the commandline
|
|
new_argv = ["-u", args.url,
|
|
"-f", str(args.factor)]
|
|
if args.force_metadata:
|
|
new_argv.extend(["--force-metadata"])
|
|
new_argv.extend([info.path, new_path])
|
|
nilmtools.decimate.main(new_argv)
|
|
|
|
# Update info using the newly decimated stream
|
|
info = nilmtools.filter.get_stream_info(client, new_path)
|
|
|
|
return
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|