You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

259 lines
9.0 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmtools.copy_one
  3. import nilmtools.cleanup
  4. import nilmtools.copy_one
  5. import nilmtools.copy_wildcard
  6. import nilmtools.decimate_auto
  7. import nilmtools.decimate
  8. import nilmtools.insert
  9. import nilmtools.median
  10. import nilmtools.pipewatch
  11. import nilmtools.prep
  12. import nilmtools.sinefit
  13. import nilmtools.trainola
  14. from nose.tools import assert_raises
  15. import unittest
  16. from testutil.helpers import *
  17. import multiprocessing
  18. import traceback
  19. from urllib.request import urlopen
  20. from nilmtools.filter import ArgumentError
  21. def run_cherrypy_server(path, port, event):
  22. db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(path)
  23. server = nilmdb.server.Server(db, host="127.0.0.1",
  24. port=port, stoppable=True)
  25. server.start(blocking = True, event = event)
  26. db.close()
  27. class CommandTester():
  28. url = "http://localhost:32182/"
  29. url2 = "http://localhost:32183/"
  30. @classmethod
  31. def setup_class(cls):
  32. # Use multiprocessing with "spawn" method, so that we can
  33. # start two fully independent cherrypy instances
  34. # (needed for copy-wildcard)
  35. multiprocessing.set_start_method('spawn')
  36. events = []
  37. for (path, port) in (("tests/testdb1", 32182),
  38. ("tests/testdb2", 32183)):
  39. recursive_unlink(path)
  40. event = multiprocessing.Event()
  41. proc = multiprocessing.Process(target=run_cherrypy_server,
  42. args=(path, port, event))
  43. proc.start()
  44. events.append(event)
  45. for event in events:
  46. if not event.wait(timeout = 10):
  47. raise AssertionError("server didn't start")
  48. @classmethod
  49. def teardown_class(cls):
  50. urlopen("http://127.0.0.1:32182/exit/", timeout = 1)
  51. urlopen("http://127.0.0.1:32183/exit/", timeout = 1)
  52. def run(self, arg_string, infile=None, outfile=None):
  53. """Run a cmdline client with the specified argument string,
  54. passing the given input. Save the output and exit code."""
  55. os.environ['NILMDB_URL'] = self.url
  56. class stdio_wrapper:
  57. def __init__(self, stdin, stdout, stderr):
  58. self.io = (stdin, stdout, stderr)
  59. def __enter__(self):
  60. self.saved = ( sys.stdin, sys.stdout, sys.stderr )
  61. ( sys.stdin, sys.stdout, sys.stderr ) = self.io
  62. def __exit__(self, type, value, traceback):
  63. ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
  64. # Empty input if none provided
  65. if infile is None:
  66. infile = io.TextIOWrapper(io.BytesIO(b""))
  67. # Capture stderr
  68. errfile = io.TextIOWrapper(io.BytesIO())
  69. if outfile is None:
  70. # If no output file, capture stdout with stderr
  71. outfile = errfile
  72. with stdio_wrapper(infile, outfile, errfile) as s:
  73. try:
  74. args = shlex.split(arg_string)
  75. sys.argv[0] = "test_runner"
  76. self.main(args)
  77. sys.exit(0)
  78. except SystemExit as e:
  79. exitcode = e.code
  80. # Capture raw binary output, and also try to decode a Unicode
  81. # string copy.
  82. self.captured_binary = outfile.buffer.getvalue()
  83. try:
  84. outfile.seek(0)
  85. self.captured = outfile.read()
  86. except UnicodeDecodeError:
  87. self.captured = None
  88. self.exitcode = exitcode
  89. def ok(self, arg_string, infile = None):
  90. self.run(arg_string, infile)
  91. if self.exitcode != 0:
  92. self.dump()
  93. eq_(self.exitcode, 0)
  94. def raises(self, exception, arg_string, infile=None):
  95. try:
  96. self.run(arg_string, infile)
  97. except exception as e:
  98. self.captured = f"Got correct exception: {e!r}\n"
  99. self.captured_binary = b""
  100. return
  101. self.dump()
  102. raise AssertionError(f"Didn't raise exception {exception}")
  103. def fail(self, arg_string, infile=None,
  104. exitcode=None, require_error=True,
  105. exception=None):
  106. self.run(arg_string, infile)
  107. if exitcode is not None and self.exitcode != exitcode:
  108. # Wrong exit code
  109. self.dump()
  110. eq_(self.exitcode, exitcode)
  111. if self.exitcode == 0:
  112. # Success, when we wanted failure
  113. self.dump()
  114. ne_(self.exitcode, 0)
  115. # Make sure the output contains the word "error" at the
  116. # beginning of a line, if requested
  117. if require_error and not re.search("^(?:test_runner: )error",
  118. self.captured, re.MULTILINE):
  119. raise AssertionError("command failed, but output doesn't "
  120. "contain the string 'error'")
  121. def contain(self, checkstring):
  122. in_(checkstring, self.captured)
  123. def match(self, checkstring):
  124. eq_(checkstring, self.captured)
  125. def matchfile(self, file):
  126. # Captured data should match file contents exactly
  127. with open(file) as f:
  128. contents = f.read()
  129. if contents != self.captured:
  130. print("--- reference file (first 1000 bytes):\n")
  131. print(contents[0:1000] + "\n")
  132. print("--- captured data (first 1000 bytes):\n")
  133. print(self.captured[0:1000] + "\n")
  134. zipped = itertools.zip_longest(contents, self.captured)
  135. for (n, (a, b)) in enumerate(zipped):
  136. if a != b:
  137. print("--- first difference is at offset", n)
  138. print("--- reference:", repr(a))
  139. print("--- captured:", repr(b))
  140. break
  141. raise AssertionError("captured data doesn't match " + file)
  142. def matchfilecount(self, file):
  143. # Last line of captured data should match the number of
  144. # non-commented lines in file
  145. count = 0
  146. with open(file) as f:
  147. for line in f:
  148. if line[0] != '#':
  149. count += 1
  150. eq_(self.captured.splitlines()[-1], sprintf("%d", count))
  151. def dump(self):
  152. printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
  153. class TestAllCommands(CommandTester):
  154. def load_data(self):
  155. client = nilmdb.client.Client(url=self.url)
  156. client.stream_create("/newton/prep", "float32_8")
  157. client.stream_set_metadata("/newton/prep",
  158. { "description": "newton" })
  159. for ts in ("20120323T1000", "20120323T1002", "20120323T1004"):
  160. start = nilmdb.utils.time.parse_time(ts)
  161. fn = f"tests/data/prep-{ts}"
  162. data = nilmdb.utils.timestamper.TimestamperRate(fn, start, 120)
  163. client.stream_insert("/newton/prep", data);
  164. def test_copy(self):
  165. self.main = nilmtools.copy_one.main
  166. client = nilmdb.client.Client(url=self.url)
  167. self.load_data()
  168. # basic arguments
  169. self.fail(f"")
  170. self.raises(ArgumentError, f"no-such-src no-such-dest")
  171. self.raises(ArgumentError, f"-u {self.url} no-such-src no-such-dest")
  172. # nonexistent dest
  173. self.fail(f"/newton/prep /newton/prep-copy", require_error=False)
  174. self.contain("Destination /newton/prep-copy doesn't exist")
  175. # wrong type
  176. client.stream_create("/newton/prep-copy-wrongtype", "uint16_6")
  177. self.raises(ValueError, f"/newton/prep /newton/prep-copy-wrongtype")
  178. # copy with metadata, and compare
  179. client.stream_create("/newton/prep-copy", "float32_8")
  180. self.ok(f"/newton/prep /newton/prep-copy")
  181. a = list(client.stream_extract("/newton/prep"))
  182. b = list(client.stream_extract("/newton/prep-copy"))
  183. eq_(a, b)
  184. a = client.stream_get_metadata("/newton/prep")
  185. b = client.stream_get_metadata("/newton/prep-copy")
  186. eq_(a, b)
  187. # copy with no metadata
  188. client.stream_create("/newton/prep-copy-nometa", "float32_8")
  189. self.ok(f"--nometa /newton/prep /newton/prep-copy-nometa")
  190. a = list(client.stream_extract("/newton/prep"))
  191. b = list(client.stream_extract("/newton/prep-copy-nometa"))
  192. eq_(a, b)
  193. a = client.stream_get_metadata("/newton/prep")
  194. b = client.stream_get_metadata("/newton/prep-copy-nometa")
  195. ne_(a, b)
  196. def test_copy_wildcard(self):
  197. self.main = nilmtools.copy_wildcard.main
  198. def test_decimate(self):
  199. self.main = nilmtools.decimate.main
  200. def test_decimate_auto(self):
  201. self.main = nilmtools.decimate_auto.main
  202. def test_insert(self):
  203. self.main = nilmtools.insert.main
  204. def test_sinefit(self):
  205. self.main = nilmtools.sinefit.main
  206. def test_cleanup(self):
  207. self.main = nilmtools.cleanup.main
  208. def test_median(self):
  209. self.main = nilmtools.median.main
  210. def test_trainola(self):
  211. self.main = nilmtools.trainola.main
  212. def test_pipewatch(self):
  213. self.main = nilmtools.pipewatch.main
  214. def test_prep(self):
  215. self.main = nilmtools.prep.main