Browse Source

Bugfixes, improvements

tags/nilmtools-0.1
Jim Paris 11 years ago
parent
commit
cd8eb5c727
1 changed files with 34 additions and 10 deletions
  1. +34
    -10
      insert.py

+ 34
- 10
insert.py View File

@@ -8,11 +8,12 @@ from nilmdb.utils.time import parse_time, format_time
import sys import sys
import re import re
import argparse import argparse
import subprocess


class ParseError(Exception): class ParseError(Exception):
def __init__(self, filename, linenum, error): def __init__(self, filename, linenum, error):
s = filename + ":" + str(linenum) + ": " + error
super(ParseError, self).__init__(error)
msg = filename + ":" + str(linenum+1) + ": " + error
super(ParseError, self).__init__(msg)


parser = argparse.ArgumentParser(description = """\ parser = argparse.ArgumentParser(description = """\
Insert data from ethstream, either live (using the system time as a Insert data from ethstream, either live (using the system time as a
@@ -49,13 +50,23 @@ data_ts = None
# comments, or or system clock) # comments, or or system clock)
clock_ts = None clock_ts = None


lines_ok = 0

with client.stream_insert_context(args.path) as stream: with client.stream_insert_context(args.path) as stream:
for f in args.infile: for f in args.infile:
printf("Processing %s\n", f.name)
filename = f.name
printf("Processing %s\n", filename)

# If the filename ends in .gz, open it with gzcat just to be
# nice.
if filename.endswith(".gz"):
p = subprocess.Popen(["gzip", "-dc"],
stdin = f, stdout = subprocess.PIPE)
f = p.stdout


# Try to get a real timestamp from the filename # Try to get a real timestamp from the filename
try: try:
clock_ts = parse_time(f.name).totimestamp()
clock_ts = parse_time(filename).totimestamp()
printf("Clock time updated to %s\n", format_time(clock_ts)) printf("Clock time updated to %s\n", format_time(clock_ts))
except ValueError: except ValueError:
pass pass
@@ -64,7 +75,7 @@ with client.stream_insert_context(args.path) as stream:
for (linenum, line) in enumerate(f): for (linenum, line) in enumerate(f):


# Any comments? # Any comments?
if line.contains('#'):
if line.find('#') >= 0:
(line, comment) = line.split('#', 1) (line, comment) = line.split('#', 1)
line += '\n' line += '\n'
else: else:
@@ -87,9 +98,9 @@ with client.stream_insert_context(args.path) as stream:
# Make a timestamp for this line # Make a timestamp for this line
if data_ts is None: if data_ts is None:
if clock_ts is None: if clock_ts is None:
err = "No idea what timestamp to use for the first line"
raise ParseError(f.name, linenum, err)
clock_ts = data_ts
err = "No idea what timestamp to use"
raise ParseError(filename, linenum, err)
data_ts = clock_ts
else: else:
data_ts += 1.0 / args.rate data_ts += 1.0 / args.rate


@@ -103,7 +114,7 @@ with client.stream_insert_context(args.path) as stream:
err = sprintf("Data is coming in too fast: data time is %s " err = sprintf("Data is coming in too fast: data time is %s "
"but clock time is only %s", "but clock time is only %s",
format_time(data_ts), format_time(clock_ts)) format_time(data_ts), format_time(clock_ts))
raise ParserError(f.name, linenum, err)
raise ParserError(filename, linenum, err)


if (data_ts + 10) < clock_ts: if (data_ts + 10) < clock_ts:
# Accumulated line timetamps are in the past. We # Accumulated line timetamps are in the past. We
@@ -118,4 +129,17 @@ with client.stream_insert_context(args.path) as stream:
# Don't use this clock time anymore until we update it # Don't use this clock time anymore until we update it
clock_ts = None clock_ts = None


stream.insert_line(line)
# Once in a while a line might be truncated, if we're at
# the end of a file. Accept it, but only if we've been
# getting good lines too
if line[-1] != '\n':
if lines_ok > 10:
printf("Ignoring short line at %s:%d\n", filename, linenum)
continue
else:
raise ParserError(filename, linenum, "short line")
lines_ok = 0

stream.insert_line(repr(data_ts) + " " + line)
lines_ok += 1
print "Done"

Loading…
Cancel
Save