Browse Source

Add nilm-insert test, and update for Python 3

tags/nilmtools-2.0.0
Jim Paris 1 year ago
parent
commit
3a8c04e04a
12 changed files with 160 additions and 19 deletions
  1. +18
    -18
      nilmtools/insert.py
  2. BIN
      tests/data/bpnilm-raw-1.gz
  3. BIN
      tests/data/bpnilm-raw-2.gz
  4. +12
    -0
      tests/data/prep-20120323T1004-delta
  5. +12
    -0
      tests/data/prep-20120323T1004-delta2
  6. +3
    -0
      tests/data/prep-20120323T1004-delta3
  7. +3
    -0
      tests/data/prep-notime
  8. +1
    -0
      tests/data/trunc1
  9. +1
    -0
      tests/data/trunc2
  10. +5
    -0
      tests/data/trunc3
  11. +1
    -0
      tests/data/trunc4
  12. +104
    -1
      tests/test.py

+ 18
- 18
nilmtools/insert.py View File

@@ -6,6 +6,7 @@ from nilmdb.utils.time import (parse_time, timestamp_to_human,
timestamp_to_seconds, seconds_to_timestamp,
rate_to_period, now as time_now)

import os
import nilmtools
import time
import sys
@@ -22,7 +23,6 @@ class ParseError(Exception):
def parse_args(argv = None):
parser = argparse.ArgumentParser(
formatter_class = argparse.RawDescriptionHelpFormatter,
version = nilmtools.__version__,
description = textwrap.dedent("""\
Insert large amount of data from an external source like ethstream.

@@ -56,9 +56,12 @@ def parse_args(argv = None):
error is raised. If '--skip' is specified, the current file
is skipped instead of raising an error.
"""))
parser.add_argument("-u", "--url", action="store",
default="http://localhost/nilmdb/",
def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
parser.add_argument("-u", "--url", action="store", default=def_url,
help="NilmDB server URL (default: %(default)s)")
parser.add_argument("-v", "--version", action="version",
version=nilmtools.__version__)

group = parser.add_argument_group("Misc options")
group.add_argument("-D", "--dry-run", action="store_true",
help="Parse files, but don't insert any data")
@@ -98,7 +101,7 @@ def parse_args(argv = None):
help="Path of stream, e.g. /foo/bar")

group = parser.add_argument_group("Input files")
group.add_argument("infile", type=argparse.FileType('r'), nargs='*',
group.add_argument("infile", type=argparse.FileType('rb'), nargs='*',
default=[sys.stdin],
help="Input files (default: stdin)")

@@ -171,7 +174,7 @@ def main(argv = None):
# decompress.
if filename.endswith(".gz"):
p = subprocess.Popen(["gzip", "-dc"],
stdin = f, stdout = subprocess.PIPE)
stdin=f, stdout=subprocess.PIPE)
f = p.stdout

# Try to get a real timestamp from the filename
@@ -184,17 +187,11 @@ def main(argv = None):
except ValueError:
pass

truncated_lines = 0

# Read each line
for line in f:
# Once in a while a line might be truncated, if we're
# at the end of a file. Ignore it, but if we ignore
# too many, bail out.
if line[-1] != '\n':
truncated_lines += 1
if truncated_lines > 3:
raise ParseError(filename, "too many short lines")
# The last line in the file may be truncated.
# Ignore it; we shouldn't ever see more than one at the end.
if line[-1] != b'\n'[0]:
printf("Ignoring short line in %s\n", filename)
continue

@@ -203,13 +200,16 @@ def main(argv = None):
continue

# If line starts with a comment, look for a timestamp
if line[0] == '#':
if line[0] == b'#'[0]:
try:
clock_ts = parse_time(line[1:]) + offset_comment
comment = line[1:].decode('utf-8', errors='ignore')
clock_ts = parse_time(comment) + offset_comment
print_clock_updated()
except ValueError:
pass
continue
# for some reason the following line doesn't show up as
# being covered, even though it definitely runs
continue # pragma: no cover

# If --delta mode, increment data_ts_delta by the
# delta from the file.
@@ -268,7 +268,7 @@ def main(argv = None):

# Insert it
if not args.dry_run:
stream.insert("%d %s" % (data_ts, line))
stream.insert(b"%d %s" % (data_ts, line))
print("Done")

if __name__ == "__main__":


BIN
tests/data/bpnilm-raw-1.gz View File


BIN
tests/data/bpnilm-raw-2.gz View File


+ 12
- 0
tests/data/prep-20120323T1004-delta View File

@@ -0,0 +1,12 @@
# deltas are in microseconds
1000000 2.61246e+05 2.22735e+05 4.60340e+03 2.58221e+03 8.42804e+03 3.41890e+03 9.57898e+02 4.00585e+03
1000000 2.56439e+05 2.24775e+05 2.92897e+03 4.66646e+03 7.58491e+03 3.57351e+03 -4.34171e+02 2.98819e+03
1000000 2.51903e+05 2.23202e+05 4.23696e+03 3.49363e+03 8.53493e+03 4.29416e+03 8.49573e+02 2.38189e+03
1000000 2.57625e+05 2.20247e+05 5.47017e+03 1.35872e+03 9.18903e+03 4.56136e+03 2.65599e+03 2.60912e+03
1000000 2.63375e+05 2.20706e+05 4.51842e+03 1.80758e+03 8.17208e+03 4.17463e+03 2.57884e+03 3.32848e+03
1000000 2.59221e+05 2.22346e+05 2.98879e+03 3.66264e+03 6.87274e+03 3.94223e+03 1.25928e+03 3.51786e+03
50000000 2.51918e+05 2.22281e+05 4.22677e+03 2.84764e+03 7.78323e+03 3.81659e+03 8.04944e+02 3.46314e+03
1000000 2.54478e+05 2.21701e+05 5.61366e+03 1.02262e+03 9.26581e+03 3.50152e+03 1.29331e+03 3.07271e+03
1000000 2.59568e+05 2.22945e+05 4.97190e+03 1.28250e+03 8.62081e+03 4.06316e+03 1.85717e+03 2.61990e+03
1000000 2.57269e+05 2.23697e+05 3.60527e+03 3.05749e+03 7.22363e+03 4.90330e+03 1.93736e+03 2.35357e+03
1000000 2.52274e+05 2.21438e+05 5.01228e+03 2.86309e+03 7.87115e+03 4.80448e+03 2.18291e+03 2.93397e+03

+ 12
- 0
tests/data/prep-20120323T1004-delta2 View File

@@ -0,0 +1,12 @@
# deltas are in microseconds
1000000 2.61246e+05 2.22735e+05 4.60340e+03 2.58221e+03 8.42804e+03 3.41890e+03 9.57898e+02 4.00585e+03
1000000 2.56439e+05 2.24775e+05 2.92897e+03 4.66646e+03 7.58491e+03 3.57351e+03 -4.34171e+02 2.98819e+03
1000000 2.51903e+05 2.23202e+05 4.23696e+03 3.49363e+03 8.53493e+03 4.29416e+03 8.49573e+02 2.38189e+03
1000000 2.57625e+05 2.20247e+05 5.47017e+03 1.35872e+03 9.18903e+03 4.56136e+03 2.65599e+03 2.60912e+03
1000000 2.63375e+05 2.20706e+05 4.51842e+03 1.80758e+03 8.17208e+03 4.17463e+03 2.57884e+03 3.32848e+03
1000000 2.59221e+05 2.22346e+05 2.98879e+03 3.66264e+03 6.87274e+03 3.94223e+03 1.25928e+03 3.51786e+03
1000000 2.51918e+05 2.22281e+05 4.22677e+03 2.84764e+03 7.78323e+03 3.81659e+03 8.04944e+02 3.46314e+03
1000000 2.54478e+05 2.21701e+05 5.61366e+03 1.02262e+03 9.26581e+03 3.50152e+03 1.29331e+03 3.07271e+03
1000000 2.59568e+05 2.22945e+05 4.97190e+03 1.28250e+03 8.62081e+03 4.06316e+03 1.85717e+03 2.61990e+03
1000000 2.57269e+05 2.23697e+05 3.60527e+03 3.05749e+03 7.22363e+03 4.90330e+03 1.93736e+03 2.35357e+03
1000000 2.52274e+05 2.21438e+05 5.01228e+03 2.86309e+03 7.87115e+03 4.80448e+03 2.18291e+03 2.93397e+03

+ 3
- 0
tests/data/prep-20120323T1004-delta3 View File

@@ -0,0 +1,3 @@
# deltas are in microseconds
1000000A 2.61246e+05 2.22735e+05 4.60340e+03 2.58221e+03 8.42804e+03 3.41890e+03 9.57898e+02 4.00585e+03
BAD_DELTA 2.56439e+05 2.24775e+05 2.92897e+03 4.66646e+03 7.58491e+03 3.57351e+03 -4.34171e+02 2.98819e+03

+ 3
- 0
tests/data/prep-notime View File

@@ -0,0 +1,3 @@
# comments are cool? what if they contain →UNICODEâ†� or invalid utf-8 like Ã(
2.66568e+05 2.24029e+05 5.16140e+03 2.52517e+03 8.35084e+03 3.72470e+03 1.35534e+03 2.03900e+03
2.57914e+05 2.27183e+05 4.30368e+03 4.13080e+03 7.25535e+03 4.89047e+03 1.63859e+03 1.93496e+03

+ 1
- 0
tests/data/trunc1 View File

@@ -0,0 +1 @@
1

+ 1
- 0
tests/data/trunc2 View File

@@ -0,0 +1 @@
1 2

+ 5
- 0
tests/data/trunc3 View File

@@ -0,0 +1,5 @@
# this has blank lines

# and timestamps: 2000-01-01 12:00:00
# and then the last is truncated
1 2 3

+ 1
- 0
tests/data/trunc4 View File

@@ -0,0 +1 @@
1 2 3 4

+ 104
- 1
tests/test.py View File

@@ -64,6 +64,7 @@ class CommandTester():
"""Run a cmdline client with the specified argument string,
passing the given input. Save the output and exit code."""
os.environ['NILMDB_URL'] = self.url
self.last_args = arg_string
class stdio_wrapper:
def __init__(self, stdin, stdout, stderr):
self.io = (stdin, stdout, stderr)
@@ -158,7 +159,8 @@ class CommandTester():
eq_(self.captured.splitlines()[-1], sprintf("%d", count))

def dump(self):
printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
printf("\n===args start===\n%s\n===args end===\n", self.last_args)
printf("===dump start===\n%s===dump end===\n", self.captured)


class TestAllCommands(CommandTester):
@@ -367,6 +369,107 @@ class TestAllCommands(CommandTester):
def test_05_insert(self):
self.main = nilmtools.insert.main

client = nilmdb.client.Client(url=self.url)

self.fail(f"")
self.ok(f"--help")

# mutually exclusive arguments
self.fail(f"--delta --rate 123 /foo bar")
self.fail(f"--live --filename /foo bar")

# Insert from file
client.stream_create("/insert/prep", "float32_8")

t0 = "tests/data/prep-20120323T1000"
t2 = "tests/data/prep-20120323T1002"
t4 = "tests/data/prep-20120323T1004"
self.ok(f"--file --dry-run --rate 120 /insert/prep {t0} {t2} {t4}")
self.contain("Dry run")

# wrong rate
self.fail(f"--file --dry-run --rate 10 /insert/prep {t0} {t2} {t4}")
self.contain("Data is coming in too fast")

# skip forward in time
self.ok(f"--file --dry-run --rate 120 /insert/prep {t0} {t4}")
self.contain("data timestamp behind by 120")
self.contain("Skipping data timestamp forward")

# skip backwards in time
self.fail(f"--file --dry-run --rate 120 /insert/prep {t0} {t2} {t0}")
self.contain("data timestamp ahead by 240")

# skip backwards in time is OK if --skip provided
self.ok(f"--skip -f -D -r 120 insert/prep {t0} {t2} {t0} {t4}")
self.contain("Skipping the remainder of this file")

# Now insert for real
self.ok(f"--skip --file --rate 120 /insert/prep {t0} {t2} {t4}")
self.contain("Done")

# Overlap
self.fail(f"--skip --file --rate 120 /insert/prep {t0}")
self.contain("new data overlaps existing data")

# Not overlap if we change file offset
self.ok(f"--skip --file --rate 120 -o 0 /insert/prep {t0}")

# Data with no timestamp
self.fail(f"-f -r 120 /insert/prep tests/data/prep-notime")
self.contain("No idea what timestamp to use")

# Check intervals so far
eq_(list(client.stream_intervals("/insert/prep")),
[[1332507600000000, 1332507959991668],
[1332511200000000, 1332511319991668]])

# Delta supplied by file
self.ok(f"--file --delta -o 0 /insert/prep {t4}-delta")
eq_(list(client.stream_intervals("/insert/prep")),
[[1332507600000000, 1332507959991668],
[1332511200000000, 1332511319991668],
[1332511440000000, 1332511499000001]])

# Now fake live timestamps by using the delta file, and a
# fake clock that increments one second per call.
def fake_time_now():
nonlocal fake_time_base
ret = fake_time_base
fake_time_base += 1000000
return ret
real_time_now = nilmtools.insert.time_now
nilmtools.insert.time_now = fake_time_now

# Delta supplied by file. This data is too fast because delta
# contains a 50 sec jump
fake_time_base = 1332511560000000
self.fail(f"--live --delta -o 0 /insert/prep {t4}-delta")
self.contain("Data is coming in too fast")
self.contain("data time is Fri, 23 Mar 2012 10:06:55")
self.contain("clock time is only Fri, 23 Mar 2012 10:06:06")

# This data is OK, no jump
fake_time_base = 1332511560000000
self.ok(f"--live --delta -o 0 /insert/prep {t4}-delta2")

# This has unparseable delta
fake_time_base = 1332511560000000
self.fail(f"--live --delta -o 0 /insert/prep {t4}-delta3")
self.contain("can't parse delta")

# Insert some gzipped data, with no timestamp in name
bp1 = "tests/data/bpnilm-raw-1.gz"
bp2 = "tests/data/bpnilm-raw-2.gz"
client.stream_create("/insert/raw", "uint16_6")
self.ok(f"--file /insert/raw {bp1} {bp2}")

# Try truncated data
tr = "tests/data/trunc"
self.ok(f"--file /insert/raw {tr}1 {tr}2 {tr}3 {tr}4")

nilmtools.insert.time_now = real_time_now

def test_06_sinefit(self):
self.main = nilmtools.sinefit.main



Loading…
Cancel
Save