@@ -1,3 +1,4 @@ | |||
.coverage | |||
oldprep | |||
newprep | |||
*.dat | |||
@@ -9,7 +9,7 @@ import nilmdb.client | |||
import nilmdb.client.numpyclient | |||
import nilmtools | |||
import argparse | |||
import ConfigParser | |||
import configparser | |||
import sys | |||
import collections | |||
import fnmatch | |||
@@ -107,7 +107,7 @@ def main(argv = None): | |||
args = parser.parse_args(argv) | |||
# Parse config file | |||
config = ConfigParser.RawConfigParser() | |||
config = configparser.RawConfigParser() | |||
config.readfp(args.configfile) | |||
# List all streams | |||
@@ -125,7 +125,7 @@ def main(argv = None): | |||
# Build up configuration | |||
for section in config.sections(): | |||
matched = False | |||
for path in streams.iterkeys(): | |||
for path in streams.keys(): | |||
# Decimated streams only allow exact matches | |||
if streams[path].decimated_from and path != section: | |||
continue | |||
@@ -157,7 +157,7 @@ def main(argv = None): | |||
warn("config for '%s' did not match any existing streams", section) | |||
# List all decimated streams in the parent stream's info | |||
for path in streams.keys(): | |||
for path in list(streams.keys()): | |||
src = streams[path].decimated_from | |||
if src and src in streams: | |||
if streams[src].clean_decimated: | |||
@@ -165,7 +165,7 @@ def main(argv = None): | |||
del streams[path] | |||
# Warn about streams that aren't getting cleaned up | |||
for path in streams.keys(): | |||
for path in list(streams.keys()): | |||
if streams[path].keep is None or streams[path].keep.seconds() < 0: | |||
warn("no config for existing stream '%s'", path) | |||
del streams[path] | |||
@@ -173,7 +173,7 @@ def main(argv = None): | |||
if args.estimate: | |||
# Estimate disk usage | |||
total = 0 | |||
for path in streams.keys(): | |||
for path in list(streams.keys()): | |||
rate = streams[path].rate | |||
if not rate or rate < 0: | |||
warn("unable to estimate disk usage for stream '%s' because " | |||
@@ -19,11 +19,11 @@ def main(argv = None): | |||
try: | |||
args = f.parse_args(argv) | |||
except nilmtools.filter.MissingDestination as e: | |||
print "Source is %s (%s)" % (e.src.path, e.src.layout) | |||
print "Destination %s doesn't exist" % (e.dest.path) | |||
print "You could make it with a command like:" | |||
print " nilmtool -u %s create %s %s" % (e.dest.url, | |||
e.dest.path, e.src.layout) | |||
print("Source is %s (%s)" % (e.src.path, e.src.layout)) | |||
print("Destination %s doesn't exist" % (e.dest.path)) | |||
print("You could make it with a command like:") | |||
print(" nilmtool -u %s create %s %s" % (e.dest.url, | |||
e.dest.path, e.src.layout)) | |||
raise SystemExit(1) | |||
# Copy metadata | |||
@@ -35,7 +35,7 @@ def main(argv = None): | |||
extractor = NumpyClient(f.src.url).stream_extract_numpy | |||
inserter = NumpyClient(f.dest.url).stream_insert_numpy_context | |||
for i in f.intervals(): | |||
print "Processing", i.human_string() | |||
print("Processing", i.human_string()) | |||
with inserter(f.dest.path, i.start, i.end) as insert_ctx: | |||
for data in extractor(f.src.path, i.start, i.end): | |||
insert_ctx.insert(data) | |||
@@ -29,8 +29,8 @@ def main(argv = None): | |||
client_dest = nilmdb.client.Client(args.dest_url) | |||
if client_src.geturl() == client_dest.geturl(): | |||
parser.error("source and destination URL must be different") | |||
print "Source URL:", client_src.geturl() | |||
print " Dest URL:", client_dest.geturl() | |||
print("Source URL:", client_src.geturl()) | |||
print(" Dest URL:", client_dest.geturl()) | |||
# Find matching streams | |||
matched = [] | |||
@@ -44,7 +44,7 @@ def main(argv = None): | |||
src = nilmtools.filter.StreamInfo(client_src.geturl(), stream) | |||
dest = nilmtools.filter.get_stream_info(client_dest, src.path) | |||
if not dest: | |||
print "Creating destination stream", src.path | |||
print("Creating destination stream", src.path) | |||
client_dest.stream_create(src.path, src.layout) | |||
# Copy them all by running the "copy" tool as if it were | |||
@@ -20,17 +20,17 @@ def main(argv = None): | |||
# a recommended layout. | |||
src = e.src | |||
dest = e.dest | |||
print "Source is %s (%s)" % (src.path, src.layout) | |||
print "Destination %s doesn't exist" % (dest.path) | |||
print("Source is %s (%s)" % (src.path, src.layout)) | |||
print("Destination %s doesn't exist" % (dest.path)) | |||
if "decimate_source" in f.client_src.stream_get_metadata(src.path): | |||
rec = src.layout | |||
elif 'int32' in src.layout_type or 'float64' in src.layout_type: | |||
rec = 'float64_' + str(src.layout_count * 3) | |||
else: | |||
rec = 'float32_' + str(src.layout_count * 3) | |||
print "You could make it with a command like:" | |||
print " nilmtool -u %s create %s %s" % (e.dest.url, | |||
e.dest.path, rec) | |||
print("You could make it with a command like:") | |||
print(" nilmtool -u %s create %s %s" % (e.dest.url, | |||
e.dest.path, rec)) | |||
raise SystemExit(1) | |||
if not (args.factor >= 2): | |||
@@ -35,13 +35,13 @@ def main(argv = None): | |||
client = nilmdb.client.Client(args.url) | |||
# Find list of paths to process | |||
streams = [ unicode(s[0]) for s in client.stream_list() ] | |||
streams = [ str(s[0]) for s in client.stream_list() ] | |||
streams = [ s for s in streams if "~decim-" not in s ] | |||
paths = [] | |||
for path in args.path: | |||
new = fnmatch.filter(streams, unicode(path)) | |||
new = fnmatch.filter(streams, str(path)) | |||
if not new: | |||
print "error: no stream matched path:", path | |||
print("error: no stream matched path:", path) | |||
raise SystemExit(1) | |||
paths.extend(new) | |||
@@ -49,15 +49,15 @@ def main(argv = None): | |||
do_decimation(client, args, path) | |||
def do_decimation(client, args, path): | |||
print "Decimating", path | |||
print("Decimating", path) | |||
info = nilmtools.filter.get_stream_info(client, path) | |||
if not info: | |||
raise Exception("path " + path + " not found") | |||
meta = client.stream_get_metadata(path) | |||
if "decimate_source" in meta: | |||
print "Stream", path, "was decimated from", meta["decimate_source"] | |||
print "You need to pass the base stream instead" | |||
print("Stream", path, "was decimated from", meta["decimate_source"]) | |||
print("You need to pass the base stream instead") | |||
raise SystemExit(1) | |||
# Figure out the type we should use for decimated streams | |||
@@ -69,7 +69,7 @@ def do_decimation(client, args, path): | |||
# Now do the decimations until we have few enough points | |||
factor = 1 | |||
while True: | |||
print "Level", factor, "decimation has", info.rows, "rows" | |||
print("Level", factor, "decimation has", info.rows, "rows") | |||
if info.rows <= 500: | |||
break | |||
factor *= args.factor | |||
@@ -78,7 +78,7 @@ def do_decimation(client, args, path): | |||
# Create the stream if needed | |||
new_info = nilmtools.filter.get_stream_info(client, new_path) | |||
if not new_info: | |||
print "Creating stream", new_path | |||
print("Creating stream", new_path) | |||
client.stream_create(new_path, decimated_type) | |||
# Run the decimation as if it were run from the commandline | |||
@@ -1,6 +1,6 @@ | |||
#!/usr/bin/python | |||
from __future__ import absolute_import | |||
import nilmdb.client | |||
from nilmdb.client import Client | |||
@@ -18,7 +18,7 @@ import sys | |||
import re | |||
import argparse | |||
import numpy as np | |||
import cStringIO | |||
import io | |||
import functools | |||
class ArgumentError(Exception): | |||
@@ -45,7 +45,7 @@ class StreamInfo(object): | |||
self.timestamp_max = info[3] | |||
self.rows = info[4] | |||
self.seconds = nilmdb.utils.time.timestamp_to_seconds(info[5]) | |||
except IndexError, TypeError: | |||
except IndexError as TypeError: | |||
pass | |||
def string(self, interhost): | |||
@@ -259,8 +259,8 @@ class Filter(object): | |||
# Print info | |||
if not quiet: | |||
print "Source:", self.src.string(self._interhost) | |||
print " Dest:", self.dest.string(self._interhost) | |||
print("Source:", self.src.string(self._interhost)) | |||
print(" Dest:", self.dest.string(self._interhost)) | |||
def parse_args(self, argv = None): | |||
"""Parse arguments from a command line""" | |||
@@ -272,7 +272,7 @@ class Filter(object): | |||
self._force_metadata = args.force_metadata | |||
if args.dry_run: | |||
for interval in self.intervals(): | |||
print interval.human_string() | |||
print(interval.human_string()) | |||
raise SystemExit(0) | |||
return args | |||
@@ -324,7 +324,7 @@ class Filter(object): | |||
if not self._force_metadata: | |||
for key in data: | |||
wanted = data[key] | |||
if not isinstance(wanted, basestring): | |||
if not isinstance(wanted, str): | |||
wanted = str(wanted) | |||
val = metadata.get(key, wanted) | |||
# Force UTF-8 encoding for comparison and display | |||
@@ -372,7 +372,7 @@ class Filter(object): | |||
inserter_func = functools.partial(inserter, self.dest.path) | |||
for interval in (intervals or self.intervals()): | |||
print "Processing", interval.human_string() | |||
print("Processing", interval.human_string()) | |||
process_numpy_interval(interval, extractor_func, inserter_func, | |||
rows * 3, function, args) | |||
@@ -383,7 +383,7 @@ def main(argv = None): | |||
parser = f.setup_parser() | |||
args = f.parse_args(argv) | |||
for i in f.intervals(): | |||
print "Generic filter: need to handle", i.human_string() | |||
print("Generic filter: need to handle", i.human_string()) | |||
if __name__ == "__main__": | |||
main() |
@@ -269,7 +269,7 @@ def main(argv = None): | |||
# Insert it | |||
if not args.dry_run: | |||
stream.insert("%d %s" % (data_ts, line)) | |||
print "Done" | |||
print("Done") | |||
if __name__ == "__main__": | |||
main() |
@@ -13,11 +13,11 @@ def main(argv = None): | |||
try: | |||
args = f.parse_args(argv) | |||
except nilmtools.filter.MissingDestination as e: | |||
print "Source is %s (%s)" % (e.src.path, e.src.layout) | |||
print "Destination %s doesn't exist" % (e.dest.path) | |||
print "You could make it with a command like:" | |||
print " nilmtool -u %s create %s %s" % (e.dest.url, | |||
e.dest.path, e.src.layout) | |||
print("Source is %s (%s)" % (e.src.path, e.src.layout)) | |||
print("Destination %s doesn't exist" % (e.dest.path)) | |||
print("You could make it with a command like:") | |||
print(" nilmtool -u %s create %s %s" % (e.dest.url, | |||
e.dest.path, e.src.layout)) | |||
raise SystemExit(1) | |||
meta = f.client_src.stream_get_metadata(f.src.path) | |||
@@ -14,7 +14,7 @@ import tempfile | |||
import threading | |||
import select | |||
import signal | |||
import Queue | |||
import queue | |||
import daemon | |||
def parse_args(argv = None): | |||
@@ -93,7 +93,7 @@ def pipewatch(args): | |||
stderr = None, | |||
preexec_fn = os.setpgrp) | |||
queue = Queue.Queue(maxsize = 4) | |||
queue = queue.Queue(maxsize = 4) | |||
reader = threading.Thread(target = reader_thread, | |||
args = (queue, generator.stdout.fileno())) | |||
reader.start() | |||
@@ -107,7 +107,7 @@ def pipewatch(args): | |||
if data is None: | |||
break | |||
consumer.stdin.write(data) | |||
except Queue.Empty: | |||
except queue.Empty: | |||
# Timeout: kill the generator | |||
fprintf(sys.stderr, "pipewatch: timeout\n") | |||
generator.terminate() | |||
@@ -43,14 +43,14 @@ def main(argv = None): | |||
args = f.parse_args(argv) | |||
except nilmtools.filter.MissingDestination as e: | |||
rec = "float32_%d" % (e.parsed_args.nharm * 2) | |||
print "Source is %s (%s)" % (e.src.path, e.src.layout) | |||
print "Destination %s doesn't exist" % (e.dest.path) | |||
print "You could make it with a command like:" | |||
print " nilmtool -u %s create %s %s" % (e.dest.url, e.dest.path, rec) | |||
print("Source is %s (%s)" % (e.src.path, e.src.layout)) | |||
print("Destination %s doesn't exist" % (e.dest.path)) | |||
print("You could make it with a command like:") | |||
print(" nilmtool -u %s create %s %s" % (e.dest.url, e.dest.path, rec)) | |||
raise SystemExit(1) | |||
if f.dest.layout_count != args.nharm * 2: | |||
print "error: need", args.nharm*2, "columns in destination stream" | |||
print("error: need", args.nharm*2, "columns in destination stream") | |||
raise SystemExit(1) | |||
# Check arguments | |||
@@ -38,10 +38,10 @@ def main(argv = None): | |||
args = f.parse_args(argv) | |||
except nilmtools.filter.MissingDestination as e: | |||
rec = "float32_3" | |||
print "Source is %s (%s)" % (e.src.path, e.src.layout) | |||
print "Destination %s doesn't exist" % (e.dest.path) | |||
print "You could make it with a command like:" | |||
print " nilmtool -u %s create %s %s" % (e.dest.url, e.dest.path, rec) | |||
print("Source is %s (%s)" % (e.src.path, e.src.layout)) | |||
print("Destination %s doesn't exist" % (e.dest.path)) | |||
print("You could make it with a command like:") | |||
print(" nilmtool -u %s create %s %s" % (e.dest.url, e.dest.path, rec)) | |||
raise SystemExit(1) | |||
if args.column is None or args.column < 1: | |||
@@ -30,7 +30,7 @@ def build_column_mapping(colinfo, streaminfo): | |||
columns = OrderedDict() | |||
for c in colinfo: | |||
col_num = c['index'] + 1 # skip timestamp | |||
if (c['name'] in columns.keys() or col_num in columns.values()): | |||
if (c['name'] in list(columns.keys()) or col_num in list(columns.values())): | |||
raise DataError("duplicated columns") | |||
if (c['index'] < 0 or c['index'] >= streaminfo.layout_count): | |||
raise DataError("bad column number") | |||
@@ -82,7 +82,7 @@ class Exemplar(object): | |||
# Extract just the columns that were specified in self.columns, | |||
# skipping the timestamp. | |||
extract_columns = [ value for (key, value) in self.columns.items() ] | |||
extract_columns = [ value for (key, value) in list(self.columns.items()) ] | |||
self.data = self.data[:,extract_columns] | |||
# Fix the column indices in e.columns, since we removed/reordered | |||
@@ -102,7 +102,7 @@ class Exemplar(object): | |||
def __str__(self): | |||
return sprintf("\"%s\" %s [%s] %s rows", | |||
self.name, self.stream, ",".join(self.columns.keys()), | |||
self.name, self.stream, ",".join(list(self.columns.keys())), | |||
self.count) | |||
def timestamp_to_short_human(timestamp): | |||
@@ -205,7 +205,7 @@ def trainola_matcher(data, interval, args, insert_func, final_chunk): | |||
return valid | |||
def trainola(conf): | |||
print "Trainola", nilmtools.__version__ | |||
print("Trainola", nilmtools.__version__) | |||
# Load main stream data | |||
url = conf['url'] | |||
@@ -227,7 +227,7 @@ def trainola(conf): | |||
raise DataError("destination path '" + dest_path + "' does not exist") | |||
printf("Source:\n") | |||
printf(" %s [%s]\n", src.path, ",".join(src_columns.keys())) | |||
printf(" %s [%s]\n", src.path, ",".join(list(src_columns.keys()))) | |||
printf("Destination:\n") | |||
printf(" %s (%s columns)\n", dest.path, dest.layout_count) | |||