You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

294 lines
10 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmtools.copy_one
  3. import nilmtools.cleanup
  4. import nilmtools.copy_one
  5. import nilmtools.copy_wildcard
  6. import nilmtools.decimate_auto
  7. import nilmtools.decimate
  8. import nilmtools.insert
  9. import nilmtools.median
  10. import nilmtools.pipewatch
  11. import nilmtools.prep
  12. import nilmtools.sinefit
  13. import nilmtools.trainola
  14. from nose.tools import assert_raises
  15. import unittest
  16. from testutil.helpers import *
  17. import multiprocessing
  18. import traceback
  19. from urllib.request import urlopen
  20. from nilmtools.filter import ArgumentError
  21. def run_cherrypy_server(path, port, event):
  22. db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(path)
  23. server = nilmdb.server.Server(db, host="127.0.0.1",
  24. port=port, stoppable=True)
  25. server.start(blocking = True, event = event)
  26. db.close()
  27. class CommandTester():
  28. url = "http://localhost:32182/"
  29. url2 = "http://localhost:32183/"
  30. @classmethod
  31. def setup_class(cls):
  32. # Use multiprocessing with "spawn" method, so that we can
  33. # start two fully independent cherrypy instances
  34. # (needed for copy-wildcard)
  35. multiprocessing.set_start_method('spawn')
  36. events = []
  37. for (path, port) in (("tests/testdb1", 32182),
  38. ("tests/testdb2", 32183)):
  39. recursive_unlink(path)
  40. event = multiprocessing.Event()
  41. proc = multiprocessing.Process(target=run_cherrypy_server,
  42. args=(path, port, event))
  43. proc.start()
  44. events.append(event)
  45. for event in events:
  46. if not event.wait(timeout = 10):
  47. raise AssertionError("server didn't start")
  48. @classmethod
  49. def teardown_class(cls):
  50. urlopen("http://127.0.0.1:32182/exit/", timeout = 1)
  51. urlopen("http://127.0.0.1:32183/exit/", timeout = 1)
  52. def run(self, arg_string, infile=None, outfile=None):
  53. """Run a cmdline client with the specified argument string,
  54. passing the given input. Save the output and exit code."""
  55. os.environ['NILMDB_URL'] = self.url
  56. class stdio_wrapper:
  57. def __init__(self, stdin, stdout, stderr):
  58. self.io = (stdin, stdout, stderr)
  59. def __enter__(self):
  60. self.saved = ( sys.stdin, sys.stdout, sys.stderr )
  61. ( sys.stdin, sys.stdout, sys.stderr ) = self.io
  62. def __exit__(self, type, value, traceback):
  63. ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
  64. # Empty input if none provided
  65. if infile is None:
  66. infile = io.TextIOWrapper(io.BytesIO(b""))
  67. # Capture stderr
  68. errfile = io.TextIOWrapper(io.BytesIO())
  69. if outfile is None:
  70. # If no output file, capture stdout with stderr
  71. outfile = errfile
  72. with stdio_wrapper(infile, outfile, errfile) as s:
  73. try:
  74. args = shlex.split(arg_string)
  75. sys.argv[0] = "test_runner"
  76. self.main(args)
  77. sys.exit(0)
  78. except SystemExit as e:
  79. exitcode = e.code
  80. except Exception as e:
  81. traceback.print_exc()
  82. exitcode = 1
  83. # Capture raw binary output, and also try to decode a Unicode
  84. # string copy.
  85. self.captured_binary = outfile.buffer.getvalue()
  86. try:
  87. outfile.seek(0)
  88. self.captured = outfile.read()
  89. except UnicodeDecodeError:
  90. self.captured = None
  91. self.exitcode = exitcode
  92. def ok(self, arg_string, infile = None):
  93. self.run(arg_string, infile)
  94. if self.exitcode != 0:
  95. self.dump()
  96. eq_(self.exitcode, 0)
  97. def fail(self, arg_string, infile=None, exitcode=None):
  98. self.run(arg_string, infile)
  99. if exitcode is not None and self.exitcode != exitcode:
  100. # Wrong exit code
  101. self.dump()
  102. eq_(self.exitcode, exitcode)
  103. if self.exitcode == 0:
  104. # Success, when we wanted failure
  105. self.dump()
  106. ne_(self.exitcode, 0)
  107. def contain(self, checkstring, contain=True):
  108. if contain:
  109. in_(checkstring, self.captured)
  110. else:
  111. nin_(checkstring, self.captured)
  112. def match(self, checkstring):
  113. eq_(checkstring, self.captured)
  114. def matchfile(self, file):
  115. # Captured data should match file contents exactly
  116. with open(file) as f:
  117. contents = f.read()
  118. if contents != self.captured:
  119. print("--- reference file (first 1000 bytes):\n")
  120. print(contents[0:1000] + "\n")
  121. print("--- captured data (first 1000 bytes):\n")
  122. print(self.captured[0:1000] + "\n")
  123. zipped = itertools.zip_longest(contents, self.captured)
  124. for (n, (a, b)) in enumerate(zipped):
  125. if a != b:
  126. print("--- first difference is at offset", n)
  127. print("--- reference:", repr(a))
  128. print("--- captured:", repr(b))
  129. break
  130. raise AssertionError("captured data doesn't match " + file)
  131. def matchfilecount(self, file):
  132. # Last line of captured data should match the number of
  133. # non-commented lines in file
  134. count = 0
  135. with open(file) as f:
  136. for line in f:
  137. if line[0] != '#':
  138. count += 1
  139. eq_(self.captured.splitlines()[-1], sprintf("%d", count))
  140. def dump(self):
  141. printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
  142. class TestAllCommands(CommandTester):
  143. def load_data(self):
  144. client = nilmdb.client.Client(url=self.url)
  145. client.stream_create("/newton/prep", "float32_8")
  146. client.stream_set_metadata("/newton/prep",
  147. { "description": "newton" })
  148. for ts in ("20120323T1000", "20120323T1002", "20120323T1004"):
  149. start = nilmdb.utils.time.parse_time(ts)
  150. fn = f"tests/data/prep-{ts}"
  151. data = nilmdb.utils.timestamper.TimestamperRate(fn, start, 120)
  152. client.stream_insert("/newton/prep", data);
  153. def test_01_copy(self):
  154. self.main = nilmtools.copy_one.main
  155. client = nilmdb.client.Client(url=self.url)
  156. self.load_data()
  157. # basic arguments
  158. self.fail(f"")
  159. self.fail(f"no-such-src no-such-dest")
  160. self.contain("source path no-such-src not found")
  161. self.fail(f"-u {self.url} no-such-src no-such-dest")
  162. # nonexistent dest
  163. self.fail(f"/newton/prep /newton/prep-copy")
  164. self.contain("Destination /newton/prep-copy doesn't exist")
  165. # wrong type
  166. client.stream_create("/newton/prep-copy-wrongtype", "uint16_6")
  167. self.fail(f"/newton/prep /newton/prep-copy-wrongtype")
  168. self.contain("wrong number of fields")
  169. # copy with metadata, and compare
  170. client.stream_create("/newton/prep-copy", "float32_8")
  171. self.ok(f"/newton/prep /newton/prep-copy")
  172. a = list(client.stream_extract("/newton/prep"))
  173. b = list(client.stream_extract("/newton/prep-copy"))
  174. eq_(a, b)
  175. a = client.stream_get_metadata("/newton/prep")
  176. b = client.stream_get_metadata("/newton/prep-copy")
  177. eq_(a, b)
  178. # copy with no metadata
  179. client.stream_create("/newton/prep-copy-nometa", "float32_8")
  180. self.ok(f"--nometa /newton/prep /newton/prep-copy-nometa")
  181. a = list(client.stream_extract("/newton/prep"))
  182. b = list(client.stream_extract("/newton/prep-copy-nometa"))
  183. eq_(a, b)
  184. a = client.stream_get_metadata("/newton/prep")
  185. b = client.stream_get_metadata("/newton/prep-copy-nometa")
  186. ne_(a, b)
  187. def test_02_copy_wildcard(self):
  188. self.main = nilmtools.copy_wildcard.main
  189. client1 = nilmdb.client.Client(url=self.url)
  190. client2 = nilmdb.client.Client(url=self.url2)
  191. # basic arguments
  192. self.fail(f"")
  193. self.fail(f"/newton")
  194. self.fail(f"-u {self.url} -U {self.url} /newton")
  195. self.contain("URL must be different")
  196. # no matches; silent
  197. self.ok(f"-u {self.url} -U {self.url2} /newton")
  198. self.ok(f"-u {self.url} -U {self.url2} /asdf*")
  199. self.ok(f"-u {self.url2} -U {self.url} /newton*")
  200. eq_(client2.stream_list(), [])
  201. # this won't actually copy, but will still create streams
  202. self.ok(f"-u {self.url} -U {self.url2} --dry-run /newton*")
  203. self.contain("Creating destination stream /newton/prep-copy")
  204. eq_(len(list(client2.stream_extract("/newton/prep"))), 0)
  205. # this should copy a bunch
  206. self.ok(f"-u {self.url} -U {self.url2} /newton*")
  207. self.contain("Creating destination stream /newton/prep-copy", False)
  208. eq_(client1.stream_list(), client2.stream_list())
  209. eq_(list(client1.stream_extract("/newton/prep")),
  210. list(client2.stream_extract("/newton/prep")))
  211. eq_(client1.stream_get_metadata("/newton/prep"),
  212. client2.stream_get_metadata("/newton/prep"))
  213. # repeating it is OK; it just won't recreate streams.
  214. # Let's try with --nometa too
  215. client2.stream_remove("/newton/prep")
  216. client2.stream_destroy("/newton/prep")
  217. self.ok(f"-u {self.url} -U {self.url2} --nometa /newton*")
  218. self.contain("Creating destination stream /newton/prep-copy", False)
  219. self.contain("Creating destination stream /newton/prep", True)
  220. eq_(client1.stream_list(), client2.stream_list())
  221. eq_(list(client1.stream_extract("/newton/prep")),
  222. list(client2.stream_extract("/newton/prep")))
  223. eq_(client2.stream_get_metadata("/newton/prep"), {})
  224. # fill in test cases
  225. self.ok(f"-u {self.url} -U {self.url2} -s 2010 -e 2020 -F /newton*")
  226. def test_03_decimate(self):
  227. self.main = nilmtools.decimate.main
  228. def test_04_decimate_auto(self):
  229. self.main = nilmtools.decimate_auto.main
  230. def test_05_insert(self):
  231. self.main = nilmtools.insert.main
  232. def test_06_sinefit(self):
  233. self.main = nilmtools.sinefit.main
  234. def test_07_cleanup(self):
  235. self.main = nilmtools.cleanup.main
  236. def test_08_median(self):
  237. self.main = nilmtools.median.main
  238. def test_09_trainola(self):
  239. self.main = nilmtools.trainola.main
  240. def test_10_pipewatch(self):
  241. self.main = nilmtools.pipewatch.main
  242. def test_11_prep(self):
  243. self.main = nilmtools.prep.main