Browse Source

Add nilm-prep test and updates for Python 3

tags/nilmtools-2.0.0
Jim Paris 1 year ago
parent
commit
752d326b77
2 changed files with 92 additions and 30 deletions
  1. +9
    -6
      nilmtools/prep.py
  2. +83
    -24
      tests/test.py

+ 9
- 6
nilmtools/prep.py View File

@@ -49,10 +49,6 @@ def main(argv = None):
print(" nilmtool -u %s create %s %s" % (e.dest.url, e.dest.path, rec))
raise SystemExit(1)

if f.dest.layout_count != args.nharm * 2:
print("error: need", args.nharm*2, "columns in destination stream")
raise SystemExit(1)

# Check arguments
if args.column is None or args.column < 1:
parser.error("need a column number >= 1")
@@ -68,6 +64,10 @@ def main(argv = None):
else:
rotation = args.rotate_rad or 0.0

if f.dest.layout_count != args.nharm * 2:
print("error: need", args.nharm*2, "columns in destination stream")
raise SystemExit(1)

# Check the sine fit stream
client_sinefit = nilmdb.client.Client(args.url)
sinefit = nilmtools.filter.get_stream_info(client_sinefit, args.sinepath)
@@ -111,7 +111,10 @@ def process(data, interval, args, insert_function, final):
def insert_if_nonoverlapping(data):
"""Call insert_function to insert data, but only if this
data doesn't overlap with other data that we inserted."""
if data[0][0] <= last_inserted[0]:
if data[0][0] <= last_inserted[0]: # pragma: no cover
# Getting coverage here is hard -- not sure exactly when
# it gets triggered or why this was added; probably some
# unlikely edge condition with timestamp rounding or something.
return
last_inserted[0] = data[-1][0]
insert_function(data)
@@ -180,7 +183,7 @@ def process(data, interval, args, insert_function, final):
# If we processed no data but there's lots in here, pretend we
# processed half of it.
if processed == 0 and rows > 10000:
processed = rows / 2
processed = rows // 2
printf("%s: warning: no periods found; skipping %d rows\n",
timestamp_to_human(data[0][0]), processed)
else:


+ 83
- 24
tests/test.py View File

@@ -471,19 +471,10 @@ class TestAllCommands(CommandTester):

nilmtools.insert.time_now = real_time_now

def test_06_sinefit(self):
self.main = nilmtools.sinefit.main
client = nilmdb.client.Client(url=self.url)

self.fail(f"")
self.ok(f"--help")

def generate_sine_data(self, client, path, data_sec, fs, freq):
# generate raw data
data_sec = 50
client.stream_create("/sf/raw", "uint16_2")
with client.stream_insert_context("/sf/raw") as ctx:
fs = 8000
freq = 60.0
client.stream_create(path, "uint16_2")
with client.stream_insert_context(path) as ctx:
for n in range(fs * data_sec):
t = n / fs
v = math.sin(t * 2 * math.pi * freq)
@@ -494,9 +485,19 @@ class TestAllCommands(CommandTester):
i * 32768 + 32768)
ctx.insert(line)
if 0:
for (s, e) in client.stream_intervals("/sf/raw"):
for (s, e) in client.stream_intervals(path):
print(Interval(s,e).human_string())


def test_06_sinefit(self):
self.main = nilmtools.sinefit.main
client = nilmdb.client.Client(url=self.url)

self.fail(f"")
self.ok(f"--help")

self.generate_sine_data(client, "/sf/raw", 50, 8000, 60)

client.stream_create("/sf/out-bad", "float32_4")
self.fail(f"--column 1 /sf/raw /sf/out-bad")
self.contain("wrong number of fields")
@@ -506,7 +507,7 @@ class TestAllCommands(CommandTester):
# basic run
client.stream_create("/sf/out", "float32_3")
self.ok(f"--column 1 /sf/raw /sf/out")
eq_(client.stream_count("/sf/out"), 60 * data_sec)
eq_(client.stream_count("/sf/out"), 3000)

# parameter errors
self.fail(f"--column 0 /sf/raw /sf/out")
@@ -551,7 +552,74 @@ class TestAllCommands(CommandTester):
self.ok("/newton/prep /median/1")
self.ok("--difference /newton/prep /median/2")

def test_08_cleanup(self):
def test_08_prep(self):
self.main = nilmtools.prep.main
client = nilmdb.client.Client(url=self.url)

self.fail(f"")
self.ok(f"--help")

self.fail(f"-c 2 /sf/raw /sf/out /prep/out")
self.contain("/prep/out doesn't exist")

# basic usage
client.stream_create("/prep/out", "float32_8")
self.ok(f"-c 2 /sf/raw /sf/out /prep/out")
self.contain("processed 100000")

# test arguments
self.fail(f"/sf/raw /sf/out /prep/out")
self.contain("need a column number")
self.fail(f"-c 0 /sf/raw /sf/out /prep/out")
self.contain("need a column number")
self.fail(f"-c 2 -n 3 /sf/raw /sf/out /prep/out")
self.contain("need 6 columns")
self.fail(f"-c 2 -n 0 /sf/raw /sf/out /prep/out")
self.contain("number of odd harmonics must be")
self.fail(f"-c 2 -N 0 /sf/raw /sf/out /prep/out")
self.contain("number of shifted FFTs must be")
self.ok(f"-c 2 -r 0 /sf/raw /sf/out /prep/out")
self.ok(f"-c 2 -R 0 /sf/raw /sf/out /prep/out")
self.fail(f"-c 2 -r 0 -R 0 /sf/raw /sf/out /prep/out")
self.fail(f"-c 2 /sf/raw /sf/no-sinefit-data /prep/out")
self.contain("sinefit data not found")
self.fail(f"-c 2 /sf/raw /prep/out /prep/out")
self.contain("sinefit data type is float32_8; expected float32_3")

# Limit time so only one row gets passed in
client.stream_create("/prep/tmp", "float32_8")
s = 1234567890000000
e = 1234567890000125
self.ok(f"-c 2 -s {s} -e {e} /sf/raw /sf/out /prep/tmp")

# Lower sampling rate on everything, so that the FFT doesn't
# return all the harmonics, and prep has to fill with zeros.
# Tests the "if N < (nharm * 2):" condition in prep
self.generate_sine_data(client, "/sf/raw-low", 5, 100, 60)
self.main = nilmtools.sinefit.main
client.stream_create("/sf/out-low", "float32_3")
self.ok(f"--column 1 /sf/raw-low /sf/out-low")
self.main = nilmtools.prep.main
client.stream_create("/prep/out-low", "float32_8")
self.ok(f"-c 2 /sf/raw-low /sf/out-low /prep/out-low")

# Test prep with empty sinefit data
client.stream_create("/sf/out-empty", "float32_3")
with client.stream_insert_context("/sf/out-empty",
1034567890123456,
2034567890123456):
pass
client.stream_create("/prep/out-empty", "float32_8")
self.ok(f"-c 2 /sf/raw /sf/out-empty /prep/out-empty")
self.contain("warning: no periods found; skipping")

def test_09_trainola(self):
self.main = nilmtools.trainola.main

def test_10_pipewatch(self):
self.main = nilmtools.pipewatch.main

def test_11_cleanup(self):
self.main = nilmtools.cleanup.main
client = nilmdb.client.Client(url=self.url)

@@ -580,12 +648,3 @@ class TestAllCommands(CommandTester):

self.ok(f"--yes tests/data/cleanup-nodecim.cfg")
self.ok(f"--estimate tests/data/cleanup-nodecim.cfg")

def test_09_trainola(self):
self.main = nilmtools.trainola.main

def test_10_pipewatch(self):
self.main = nilmtools.pipewatch.main

def test_11_prep(self):
self.main = nilmtools.prep.main

Loading…
Cancel
Save