You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

387 lines
14 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmtools.copy_one
  3. import nilmtools.cleanup
  4. import nilmtools.copy_one
  5. import nilmtools.copy_wildcard
  6. import nilmtools.decimate_auto
  7. import nilmtools.decimate
  8. import nilmtools.insert
  9. import nilmtools.median
  10. import nilmtools.pipewatch
  11. import nilmtools.prep
  12. import nilmtools.sinefit
  13. import nilmtools.trainola
  14. from nose.tools import assert_raises
  15. import unittest
  16. from testutil.helpers import *
  17. import multiprocessing
  18. import traceback
  19. from urllib.request import urlopen
  20. from nilmtools.filter import ArgumentError
  21. def run_cherrypy_server(path, port, event):
  22. db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(path)
  23. server = nilmdb.server.Server(db, host="127.0.0.1",
  24. port=port, stoppable=True)
  25. server.start(blocking = True, event = event)
  26. db.close()
  27. class CommandTester():
  28. url = "http://localhost:32182/"
  29. url2 = "http://localhost:32183/"
  30. @classmethod
  31. def setup_class(cls):
  32. # Use multiprocessing with "spawn" method, so that we can
  33. # start two fully independent cherrypy instances
  34. # (needed for copy-wildcard)
  35. multiprocessing.set_start_method('spawn')
  36. events = []
  37. for (path, port) in (("tests/testdb1", 32182),
  38. ("tests/testdb2", 32183)):
  39. recursive_unlink(path)
  40. event = multiprocessing.Event()
  41. proc = multiprocessing.Process(target=run_cherrypy_server,
  42. args=(path, port, event))
  43. proc.start()
  44. events.append(event)
  45. for event in events:
  46. if not event.wait(timeout = 10):
  47. raise AssertionError("server didn't start")
  48. @classmethod
  49. def teardown_class(cls):
  50. urlopen("http://127.0.0.1:32182/exit/", timeout = 1)
  51. urlopen("http://127.0.0.1:32183/exit/", timeout = 1)
  52. def run(self, arg_string, infile=None, outfile=None):
  53. """Run a cmdline client with the specified argument string,
  54. passing the given input. Save the output and exit code."""
  55. os.environ['NILMDB_URL'] = self.url
  56. class stdio_wrapper:
  57. def __init__(self, stdin, stdout, stderr):
  58. self.io = (stdin, stdout, stderr)
  59. def __enter__(self):
  60. self.saved = ( sys.stdin, sys.stdout, sys.stderr )
  61. ( sys.stdin, sys.stdout, sys.stderr ) = self.io
  62. def __exit__(self, type, value, traceback):
  63. ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
  64. # Empty input if none provided
  65. if infile is None:
  66. infile = io.TextIOWrapper(io.BytesIO(b""))
  67. # Capture stderr
  68. errfile = io.TextIOWrapper(io.BytesIO())
  69. if outfile is None:
  70. # If no output file, capture stdout with stderr
  71. outfile = errfile
  72. with stdio_wrapper(infile, outfile, errfile) as s:
  73. try:
  74. args = shlex.split(arg_string)
  75. sys.argv[0] = "test_runner"
  76. self.main(args)
  77. sys.exit(0)
  78. except SystemExit as e:
  79. exitcode = e.code
  80. except Exception as e:
  81. traceback.print_exc()
  82. exitcode = 1
  83. # Capture raw binary output, and also try to decode a Unicode
  84. # string copy.
  85. self.captured_binary = outfile.buffer.getvalue()
  86. try:
  87. outfile.seek(0)
  88. self.captured = outfile.read()
  89. except UnicodeDecodeError:
  90. self.captured = None
  91. self.exitcode = exitcode
  92. def ok(self, arg_string, infile = None):
  93. self.run(arg_string, infile)
  94. if self.exitcode != 0:
  95. self.dump()
  96. eq_(self.exitcode, 0)
  97. def fail(self, arg_string, infile=None, exitcode=None):
  98. self.run(arg_string, infile)
  99. if exitcode is not None and self.exitcode != exitcode:
  100. # Wrong exit code
  101. self.dump()
  102. eq_(self.exitcode, exitcode)
  103. if self.exitcode == 0:
  104. # Success, when we wanted failure
  105. self.dump()
  106. ne_(self.exitcode, 0)
  107. def contain(self, checkstring, contain=True):
  108. if contain:
  109. in_(checkstring, self.captured)
  110. else:
  111. nin_(checkstring, self.captured)
  112. def match(self, checkstring):
  113. eq_(checkstring, self.captured)
  114. def matchfile(self, file):
  115. # Captured data should match file contents exactly
  116. with open(file) as f:
  117. contents = f.read()
  118. if contents != self.captured:
  119. print("--- reference file (first 1000 bytes):\n")
  120. print(contents[0:1000] + "\n")
  121. print("--- captured data (first 1000 bytes):\n")
  122. print(self.captured[0:1000] + "\n")
  123. zipped = itertools.zip_longest(contents, self.captured)
  124. for (n, (a, b)) in enumerate(zipped):
  125. if a != b:
  126. print("--- first difference is at offset", n)
  127. print("--- reference:", repr(a))
  128. print("--- captured:", repr(b))
  129. break
  130. raise AssertionError("captured data doesn't match " + file)
  131. def matchfilecount(self, file):
  132. # Last line of captured data should match the number of
  133. # non-commented lines in file
  134. count = 0
  135. with open(file) as f:
  136. for line in f:
  137. if line[0] != '#':
  138. count += 1
  139. eq_(self.captured.splitlines()[-1], sprintf("%d", count))
  140. def dump(self):
  141. printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
  142. class TestAllCommands(CommandTester):
  143. def load_data(self):
  144. client = nilmdb.client.Client(url=self.url)
  145. client.stream_create("/newton/prep", "float32_8")
  146. client.stream_set_metadata("/newton/prep",
  147. { "description": "newton" })
  148. for ts in ("20120323T1000", "20120323T1002", "20120323T1004"):
  149. start = nilmdb.utils.time.parse_time(ts)
  150. fn = f"tests/data/prep-{ts}"
  151. data = nilmdb.utils.timestamper.TimestamperRate(fn, start, 120)
  152. client.stream_insert("/newton/prep", data);
  153. def test_01_copy(self):
  154. self.main = nilmtools.copy_one.main
  155. client = nilmdb.client.Client(url=self.url)
  156. self.load_data()
  157. # basic arguments
  158. self.fail(f"")
  159. self.fail(f"no-such-src no-such-dest")
  160. self.contain("source path no-such-src not found")
  161. self.fail(f"-u {self.url} no-such-src no-such-dest")
  162. # nonexistent dest
  163. self.fail(f"/newton/prep /newton/prep-copy")
  164. self.contain("Destination /newton/prep-copy doesn't exist")
  165. # wrong type
  166. client.stream_create("/newton/prep-copy-wrongtype", "uint16_6")
  167. self.fail(f"/newton/prep /newton/prep-copy-wrongtype")
  168. self.contain("wrong number of fields")
  169. # copy with metadata, and compare
  170. client.stream_create("/newton/prep-copy", "float32_8")
  171. self.ok(f"/newton/prep /newton/prep-copy")
  172. a = list(client.stream_extract("/newton/prep"))
  173. b = list(client.stream_extract("/newton/prep-copy"))
  174. eq_(a, b)
  175. a = client.stream_get_metadata("/newton/prep")
  176. b = client.stream_get_metadata("/newton/prep-copy")
  177. eq_(a, b)
  178. # copy with no metadata
  179. client.stream_create("/newton/prep-copy-nometa", "float32_8")
  180. self.ok(f"--nometa /newton/prep /newton/prep-copy-nometa")
  181. a = list(client.stream_extract("/newton/prep"))
  182. b = list(client.stream_extract("/newton/prep-copy-nometa"))
  183. eq_(a, b)
  184. a = client.stream_get_metadata("/newton/prep")
  185. b = client.stream_get_metadata("/newton/prep-copy-nometa")
  186. ne_(a, b)
  187. def test_02_copy_wildcard(self):
  188. self.main = nilmtools.copy_wildcard.main
  189. client1 = nilmdb.client.Client(url=self.url)
  190. client2 = nilmdb.client.Client(url=self.url2)
  191. # basic arguments
  192. self.fail(f"")
  193. self.fail(f"/newton")
  194. self.fail(f"-u {self.url} -U {self.url} /newton")
  195. self.contain("URL must be different")
  196. # no matches; silent
  197. self.ok(f"-u {self.url} -U {self.url2} /newton")
  198. self.ok(f"-u {self.url} -U {self.url2} /asdf*")
  199. self.ok(f"-u {self.url2} -U {self.url} /newton*")
  200. eq_(client2.stream_list(), [])
  201. # this won't actually copy, but will still create streams
  202. self.ok(f"-u {self.url} -U {self.url2} --dry-run /newton*")
  203. self.contain("Creating destination stream /newton/prep-copy")
  204. eq_(len(list(client2.stream_extract("/newton/prep"))), 0)
  205. # this should copy a bunch
  206. self.ok(f"-u {self.url} -U {self.url2} /newton*")
  207. self.contain("Creating destination stream /newton/prep-copy", False)
  208. eq_(client1.stream_list(), client2.stream_list())
  209. eq_(list(client1.stream_extract("/newton/prep")),
  210. list(client2.stream_extract("/newton/prep")))
  211. eq_(client1.stream_get_metadata("/newton/prep"),
  212. client2.stream_get_metadata("/newton/prep"))
  213. # repeating it is OK; it just won't recreate streams.
  214. # Let's try with --nometa too
  215. client2.stream_remove("/newton/prep")
  216. client2.stream_destroy("/newton/prep")
  217. self.ok(f"-u {self.url} -U {self.url2} --nometa /newton*")
  218. self.contain("Creating destination stream /newton/prep-copy", False)
  219. self.contain("Creating destination stream /newton/prep", True)
  220. eq_(client1.stream_list(), client2.stream_list())
  221. eq_(list(client1.stream_extract("/newton/prep")),
  222. list(client2.stream_extract("/newton/prep")))
  223. eq_(client2.stream_get_metadata("/newton/prep"), {})
  224. # fill in test cases
  225. self.ok(f"-u {self.url} -U {self.url2} -s 2010 -e 2020 -F /newton*")
  226. def test_03_decimate(self):
  227. self.main = nilmtools.decimate.main
  228. client = nilmdb.client.Client(url=self.url)
  229. # basic arguments
  230. self.fail(f"")
  231. # no dest
  232. self.fail(f"/newton/prep /newton/prep-decimated-1")
  233. self.contain("doesn't exist")
  234. # wrong dest shape
  235. client.stream_create("/newton/prep-decimated-bad", "float32_8")
  236. self.fail(f"/newton/prep /newton/prep-decimated-bad")
  237. self.contain("wrong number of fields")
  238. # bad factor
  239. self.fail(f"/newton/prep -f 1 /newton/prep-decimated-bad")
  240. self.contain("needs to be 2 or more")
  241. # ok, default factor 4
  242. client.stream_create("/newton/prep-decimated-4", "float32_24")
  243. self.ok(f"/newton/prep /newton/prep-decimated-4")
  244. a = client.stream_count("/newton/prep")
  245. b = client.stream_count("/newton/prep-decimated-4")
  246. eq_(a // 4, b)
  247. # factor 10
  248. client.stream_create("/newton/prep-decimated-10", "float32_24")
  249. self.ok(f"/newton/prep -f 10 /newton/prep-decimated-10")
  250. self.contain("Processing")
  251. a = client.stream_count("/newton/prep")
  252. b = client.stream_count("/newton/prep-decimated-10")
  253. eq_(a // 10, b)
  254. # different factor, same target
  255. self.fail(f"/newton/prep -f 16 /newton/prep-decimated-10")
  256. self.contain("Metadata in destination stream")
  257. self.contain("decimate_factor = 10")
  258. self.contain("doesn't match desired data")
  259. self.contain("decimate_factor = 16")
  260. # unless we force it
  261. self.ok(f"/newton/prep -f 16 -F /newton/prep-decimated-10")
  262. a = client.stream_count("/newton/prep")
  263. b = client.stream_count("/newton/prep-decimated-10")
  264. # but all data was already converted, so no more
  265. eq_(a // 10, b)
  266. # if we try to decimate an already-decimated stream, the suggested
  267. # shape is different
  268. self.fail(f"/newton/prep-decimated-4 -f 4 /newton/prep-decimated-16")
  269. self.contain("create /newton/prep-decimated-16 float32_24")
  270. # decimate again
  271. client.stream_create("/newton/prep-decimated-16", "float32_24")
  272. self.ok(f"/newton/prep-decimated-4 -f 4 /newton/prep-decimated-16")
  273. self.contain("Processing")
  274. # check shape suggestion for different input types
  275. for (shape, expected) in (("int32_1", "float64_3"),
  276. ("uint32_1", "float64_3"),
  277. ("int64_1", "float64_3"),
  278. ("uint64_1", "float64_3"),
  279. ("float32_1", "float32_3"),
  280. ("float64_1", "float64_3")):
  281. client.stream_create(f"/test/{shape}", shape)
  282. self.fail(f"/test/{shape} /test/{shape}-decim")
  283. self.contain(f"create /test/{shape}-decim {expected}")
  284. def test_04_decimate_auto(self):
  285. self.main = nilmtools.decimate_auto.main
  286. client = nilmdb.client.Client(url=self.url)
  287. self.fail(f"")
  288. self.fail(f"--max -1 asdf")
  289. self.contain("bad max")
  290. self.fail(f"/no/such/stream")
  291. self.contain("no stream matched path")
  292. # normal run
  293. self.ok(f"/newton/prep")
  294. # can't auto decimate a decimated stream
  295. self.fail(f"/newton/prep-decimated-16")
  296. self.contain("need to pass the base stream instead")
  297. # decimate prep again, this time much more; also use -F
  298. self.ok(f"-m 10 --force-metadata /newton/pr??")
  299. self.contain("Level 4096 decimation has 9 rows")
  300. # decimate the different shapes
  301. self.ok(f"/test/*")
  302. self.contain("Level 1 decimation has 0 rows")
  303. def test_05_insert(self):
  304. self.main = nilmtools.insert.main
  305. def test_06_sinefit(self):
  306. self.main = nilmtools.sinefit.main
  307. def test_07_cleanup(self):
  308. self.main = nilmtools.cleanup.main
  309. def test_08_median(self):
  310. self.main = nilmtools.median.main
  311. def test_09_trainola(self):
  312. self.main = nilmtools.trainola.main
  313. def test_10_pipewatch(self):
  314. self.main = nilmtools.pipewatch.main
  315. def test_11_prep(self):
  316. self.main = nilmtools.prep.main