You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

1246 lines
48 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb.server
  3. from nilmdb.utils.printf import *
  4. import nilmdb.cmdline
  5. import datetime_tz
  6. import unittest
  7. from nose.tools import *
  8. from nose.tools import assert_raises
  9. import itertools
  10. import os
  11. import re
  12. import sys
  13. import io
  14. import shlex
  15. import warnings
  16. from testutil.helpers import *
  17. testdb = "tests/cmdline-testdb"
  18. def server_start(max_results = None,
  19. max_removals = None,
  20. max_int_removals = None,
  21. bulkdata_args = {}):
  22. global test_server, test_db
  23. # Start web app on a custom port
  24. test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(
  25. testdb,
  26. max_results = max_results,
  27. max_removals = max_removals,
  28. max_int_removals = max_int_removals,
  29. bulkdata_args = bulkdata_args)
  30. test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
  31. port = 32180, stoppable = False,
  32. fast_shutdown = True,
  33. force_traceback = False)
  34. test_server.start(blocking = False)
  35. def server_stop():
  36. global test_server, test_db
  37. # Close web app
  38. test_server.stop()
  39. test_db.close()
  40. def setup_module():
  41. global test_server, test_db
  42. # Clear out DB
  43. recursive_unlink(testdb)
  44. server_start()
  45. def teardown_module():
  46. server_stop()
  47. class TestCmdline(object):
  48. def run(self, arg_string, infile=None, outfile=None):
  49. """Run a cmdline client with the specified argument string,
  50. passing the given input. Save the output and exit code."""
  51. # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
  52. os.environ['NILMDB_URL'] = "http://localhost:32180/"
  53. class stdio_wrapper:
  54. def __init__(self, stdin, stdout, stderr):
  55. self.io = (stdin, stdout, stderr)
  56. def __enter__(self):
  57. self.saved = ( sys.stdin, sys.stdout, sys.stderr )
  58. ( sys.stdin, sys.stdout, sys.stderr ) = self.io
  59. def __exit__(self, type, value, traceback):
  60. ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
  61. # Empty input if none provided
  62. if infile is None:
  63. infile = io.TextIOWrapper(io.BytesIO(b""))
  64. # Capture stderr
  65. errfile = io.TextIOWrapper(io.BytesIO())
  66. if outfile is None:
  67. # If no output file, capture stdout with stderr
  68. outfile = errfile
  69. with stdio_wrapper(infile, outfile, errfile) as s:
  70. try:
  71. args = shlex.split(arg_string)
  72. nilmdb.cmdline.Cmdline(args).run()
  73. sys.exit(0)
  74. except SystemExit as e:
  75. exitcode = e.code
  76. # Capture raw binary output, and also try to decode a Unicode
  77. # string copy.
  78. self.captured_binary = outfile.buffer.getvalue()
  79. try:
  80. outfile.seek(0)
  81. self.captured = outfile.read()
  82. except UnicodeDecodeError:
  83. self.captured = None
  84. self.exitcode = exitcode
  85. def ok(self, arg_string, infile = None):
  86. self.run(arg_string, infile)
  87. if self.exitcode != 0:
  88. self.dump()
  89. eq_(self.exitcode, 0)
  90. def fail(self, arg_string, infile = None,
  91. exitcode = None, require_error = True):
  92. self.run(arg_string, infile)
  93. if exitcode is not None and self.exitcode != exitcode:
  94. # Wrong exit code
  95. self.dump()
  96. eq_(self.exitcode, exitcode)
  97. if self.exitcode == 0:
  98. # Success, when we wanted failure
  99. self.dump()
  100. ne_(self.exitcode, 0)
  101. # Make sure the output contains the word "error" at the
  102. # beginning of a line, but only if an exitcode wasn't
  103. # specified.
  104. if require_error and not re.search("^error",
  105. self.captured, re.MULTILINE):
  106. raise AssertionError("command failed, but output doesn't "
  107. "contain the string 'error'")
  108. def contain(self, checkstring):
  109. in_(checkstring, self.captured)
  110. def match(self, checkstring):
  111. eq_(checkstring, self.captured)
  112. def matchfile(self, file):
  113. # Captured data should match file contents exactly
  114. with open(file) as f:
  115. contents = f.read()
  116. if contents != self.captured:
  117. print("--- reference file (first 1000 bytes):\n")
  118. print(contents[0:1000] + "\n")
  119. print("--- captured data (first 1000 bytes):\n")
  120. print(self.captured[0:1000] + "\n")
  121. zipped = itertools.zip_longest(contents, self.captured)
  122. for (n, (a, b)) in enumerate(zipped):
  123. if a != b:
  124. print("--- first difference is at offset", n)
  125. print("--- reference:", repr(a))
  126. print("--- captured:", repr(b))
  127. break
  128. raise AssertionError("captured data doesn't match " + file)
  129. def matchfilecount(self, file):
  130. # Last line of captured data should match the number of
  131. # non-commented lines in file
  132. count = 0
  133. with open(file) as f:
  134. for line in f:
  135. if line[0] != '#':
  136. count += 1
  137. eq_(self.captured.splitlines()[-1], sprintf("%d", count))
  138. def dump(self):
  139. printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
  140. def test_01_basic(self):
  141. # help
  142. self.ok("--help")
  143. self.contain("usage:")
  144. # help
  145. self.ok("--version")
  146. ver = self.captured
  147. self.ok("list --version")
  148. eq_(self.captured, ver)
  149. # fail for no args
  150. self.fail("")
  151. # fail for no such option
  152. self.fail("--nosuchoption")
  153. # fail for bad command
  154. self.fail("badcommand")
  155. # try some URL constructions
  156. self.fail("--url http://nosuchurl/ info")
  157. self.contain("error connecting to server")
  158. self.fail("--url nosuchurl info")
  159. self.contain("error connecting to server")
  160. self.fail("-u nosuchurl/foo info")
  161. self.contain("error connecting to server")
  162. self.fail("-u localhost:1 info")
  163. self.contain("error connecting to server")
  164. self.ok("-u localhost:32180 info")
  165. self.ok("info")
  166. # Duplicated arguments should fail, but this isn't implemented
  167. # due to it being kind of a pain with argparse.
  168. if 0:
  169. self.fail("-u url1 -u url2 info")
  170. self.contain("duplicated argument")
  171. self.fail("list --detail --detail")
  172. self.contain("duplicated argument")
  173. self.fail("list --detail --path path1 --path path2")
  174. self.contain("duplicated argument")
  175. self.fail("extract --start 2000-01-01 --start 2001-01-02")
  176. self.contain("duplicated argument")
  177. # Verify that "help command" and "command --help" are identical
  178. # for all commands.
  179. self.fail("")
  180. m = re.search(r"{(.*)}", self.captured)
  181. for command in [""] + m.group(1).split(','):
  182. self.ok(command + " --help")
  183. cap1 = self.captured
  184. self.ok("help " + command)
  185. cap2 = self.captured
  186. self.ok("help " + command + " asdf --url --zxcv -")
  187. cap3 = self.captured
  188. eq_(cap1, cap2)
  189. eq_(cap2, cap3)
  190. def test_02_parsetime(self):
  191. os.environ['TZ'] = "America/New_York"
  192. test = datetime_tz.datetime_tz.now()
  193. u2ts = nilmdb.utils.time.unix_to_timestamp
  194. parse_time = nilmdb.utils.time.parse_time
  195. eq_(parse_time(str(test)), u2ts(test.totimestamp()))
  196. test = u2ts(datetime_tz.datetime_tz.smartparse("20120405 1400-0400").
  197. totimestamp())
  198. eq_(parse_time("hi there 20120405 1400-0400 testing! 123"), test)
  199. eq_(parse_time("20120405 1800 UTC"), test)
  200. eq_(parse_time("20120405 1400-0400 UTC"), test)
  201. for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
  202. with assert_raises(ValueError):
  203. x = parse_time(badtime)
  204. x = parse_time("now")
  205. eq_(parse_time("snapshot-20120405-140000.raw.gz"), test)
  206. eq_(parse_time("prep-20120405T1400"), test)
  207. eq_(parse_time("1333648800.0"), test)
  208. eq_(parse_time("1333648800000000"), test)
  209. eq_(parse_time("@1333648800000000"), test)
  210. eq_(parse_time("min"), nilmdb.utils.time.min_timestamp)
  211. eq_(parse_time("max"), nilmdb.utils.time.max_timestamp)
  212. with assert_raises(ValueError):
  213. parse_time("@hashtag12345")
  214. def test_03_info(self):
  215. self.ok("info")
  216. self.contain("Server URL: http://localhost:32180/")
  217. self.contain("Client version: " + nilmdb.__version__)
  218. self.contain("Server version: " + test_server.version)
  219. self.contain("Server database path")
  220. self.contain("Server disk space used by NilmDB")
  221. self.contain("Server disk space used by other")
  222. self.contain("Server disk space reserved")
  223. self.contain("Server disk space free")
  224. def test_04_createlist(self):
  225. # Basic stream tests, like those in test_client.
  226. # No streams
  227. self.ok("list")
  228. self.match("")
  229. # Bad paths
  230. self.fail("create foo/bar/baz float32_8")
  231. self.contain("paths must start with /")
  232. self.fail("create /foo float32_8")
  233. self.contain("invalid path")
  234. self.fail("create /newton/prep/ float32_8")
  235. self.contain("invalid path")
  236. self.fail("create /newton/_format/prep float32_8")
  237. self.contain("path name is invalid")
  238. self.fail("create /_format/newton/prep float32_8")
  239. self.contain("path name is invalid")
  240. self.fail("create /newton/prep/_format float32_8")
  241. self.contain("path name is invalid")
  242. # Bad layout type
  243. self.fail("create /newton/prep NoSuchLayout")
  244. self.contain("no such layout")
  245. self.fail("create /newton/prep float32_0")
  246. self.contain("no such layout")
  247. self.fail("create /newton/prep float33_1")
  248. self.contain("no such layout")
  249. # Create a few streams
  250. self.ok("create /newton/zzz/rawnotch uint16_9")
  251. self.ok("create /newton/prep float32_8")
  252. self.ok("create /newton/raw uint16_6")
  253. self.ok("create /newton/raw~decim-1234 uint16_6")
  254. # Create a stream that already exists
  255. self.fail("create /newton/raw uint16_6")
  256. self.contain("stream already exists at this path")
  257. # Should not be able to create a stream with another stream as
  258. # its parent
  259. self.fail("create /newton/prep/blah float32_8")
  260. self.contain("path is subdir of existing node")
  261. # Should not be able to create a stream at a location that
  262. # has other nodes as children
  263. self.fail("create /newton/zzz float32_8")
  264. self.contain("subdirs of this path already exist")
  265. # Verify we got those 4 streams and they're returned in
  266. # alphabetical order.
  267. self.ok("list -l")
  268. self.match("/newton/prep float32_8\n"
  269. "/newton/raw uint16_6\n"
  270. "/newton/raw~decim-1234 uint16_6\n"
  271. "/newton/zzz/rawnotch uint16_9\n")
  272. # No decimated streams if -n specified
  273. self.ok("list -n -l")
  274. self.match("/newton/prep float32_8\n"
  275. "/newton/raw uint16_6\n"
  276. "/newton/zzz/rawnotch uint16_9\n")
  277. # Delete that decimated stream
  278. self.ok("destroy /newton/raw~decim-1234")
  279. # Match just one type or one path. Also check
  280. # that --path is optional
  281. self.ok("list --layout /newton/raw")
  282. self.match("/newton/raw uint16_6\n")
  283. # Wildcard matches
  284. self.ok("list *zzz*")
  285. self.match("/newton/zzz/rawnotch\n")
  286. # reversed range
  287. self.fail("list /newton/prep --start 2020-01-01 --end 2000-01-01")
  288. self.contain("start must precede end")
  289. def test_05_metadata(self):
  290. # Set / get metadata
  291. self.fail("metadata")
  292. self.fail("metadata --get")
  293. self.ok("metadata /newton/prep")
  294. self.match("")
  295. self.ok("metadata /newton/raw --get")
  296. self.match("")
  297. self.ok("metadata /newton/prep --set "
  298. "'description=The Data' "
  299. "v_scale=1.234")
  300. self.ok("metadata /newton/raw --update "
  301. "'description=The Data'")
  302. self.ok("metadata /newton/raw --update "
  303. "v_scale=1.234")
  304. # unicode
  305. self.ok("metadata /newton/raw --set "
  306. "a_๐“ด๐“ฎ๐”‚=value a_key=๐“ฟ๐“ช๐“ต๐“พ๐“ฎ a_๐—ธ๐—ฒ๐˜†=๐˜ƒ๐—ฎ๐—น๐˜‚๐—ฒ")
  307. self.ok("metadata /newton/raw --get")
  308. self.match("a_key=๐“ฟ๐“ช๐“ต๐“พ๐“ฎ\na_๐“ด๐“ฎ๐”‚=value\na_๐—ธ๐—ฒ๐˜†=๐˜ƒ๐—ฎ๐—น๐˜‚๐—ฒ\n")
  309. # various parsing tests
  310. self.ok("metadata /newton/raw --update foo=")
  311. self.fail("metadata /newton/raw --update =bar")
  312. self.fail("metadata /newton/raw --update foo==bar")
  313. self.fail("metadata /newton/raw --update foo;bar")
  314. # errors
  315. self.fail("metadata /newton/nosuchstream foo=bar")
  316. self.contain("unrecognized arguments")
  317. self.fail("metadata /newton/nosuchstream")
  318. self.contain("No stream at path")
  319. self.fail("metadata /newton/nosuchstream --set foo=bar")
  320. self.contain("No stream at path")
  321. self.fail("metadata /newton/nosuchstream --delete")
  322. self.contain("No stream at path")
  323. self.ok("metadata /newton/prep")
  324. self.match("description=The Data\nv_scale=1.234\n")
  325. self.ok("metadata /newton/prep --get")
  326. self.match("description=The Data\nv_scale=1.234\n")
  327. self.ok("metadata /newton/prep --get descr")
  328. self.match("descr=\n")
  329. self.ok("metadata /newton/prep --get description")
  330. self.match("description=The Data\n")
  331. self.ok("metadata /newton/prep --get description v_scale")
  332. self.match("description=The Data\nv_scale=1.234\n")
  333. self.ok("metadata /newton/prep --set "
  334. "'description=The Data'")
  335. self.ok("metadata /newton/prep --get")
  336. self.match("description=The Data\n")
  337. self.fail("metadata /newton/nosuchpath")
  338. self.contain("No stream at path /newton/nosuchpath")
  339. self.ok("metadata /newton/prep --delete")
  340. self.ok("metadata /newton/prep --get")
  341. self.match("")
  342. self.ok("metadata /newton/prep --set "
  343. "'description=The Data' "
  344. "v_scale=1.234")
  345. self.ok("metadata /newton/prep --delete v_scale")
  346. self.ok("metadata /newton/prep --get")
  347. self.match("description=The Data\n")
  348. self.ok("metadata /newton/prep --set description=")
  349. self.ok("metadata /newton/prep --get")
  350. self.match("")
  351. def test_06_insert(self):
  352. self.ok("insert --help")
  353. self.fail("insert -s 2000 -e 2001 /foo/bar baz")
  354. self.contain("error getting stream info")
  355. self.fail("insert -s 2000 -e 2001 /newton/prep baz")
  356. self.match("error opening input file baz\n")
  357. self.fail("insert /newton/prep --timestamp -f -r 120")
  358. self.contain("error extracting start time")
  359. self.fail("insert /newton/prep --timestamp -r 120")
  360. self.contain("need --start or --filename")
  361. self.fail("insert /newton/prep "
  362. "tests/data/prep-20120323T1000")
  363. # insert pre-timestamped data, with bad times (non-monotonic)
  364. os.environ['TZ'] = "UTC"
  365. with open("tests/data/prep-20120323T1004-badtimes") as input:
  366. self.fail("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  367. input)
  368. self.contain("error parsing input data")
  369. self.contain("line 7")
  370. self.contain("timestamp is not monotonically increasing")
  371. # insert pre-timestamped data, from stdin
  372. os.environ['TZ'] = "UTC"
  373. with open("tests/data/prep-20120323T1004-timestamped") as input:
  374. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  375. input)
  376. # insert data with normal timestamper from filename
  377. os.environ['TZ'] = "UTC"
  378. self.ok("insert --timestamp -f --rate 120 /newton/prep "
  379. "tests/data/prep-20120323T1000")
  380. self.fail("insert -t --filename /newton/prep "
  381. "tests/data/prep-20120323T1002")
  382. self.contain("rate is needed")
  383. self.ok("insert -t --filename --rate 120 /newton/prep "
  384. "tests/data/prep-20120323T1002")
  385. # overlap
  386. os.environ['TZ'] = "UTC"
  387. self.fail("insert --timestamp -f --rate 120 /newton/prep "
  388. "tests/data/prep-20120323T1004")
  389. self.contain("overlap")
  390. # Just to help test more situations -- stop and restart
  391. # the server now. This tests nilmdb's interval caching,
  392. # at the very least.
  393. server_stop()
  394. server_start()
  395. # still an overlap if we specify a different start
  396. os.environ['TZ'] = "America/New_York"
  397. self.fail("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep"
  398. " tests/data/prep-20120323T1004")
  399. self.contain("overlap")
  400. # wrong format
  401. os.environ['TZ'] = "UTC"
  402. self.fail("insert -t -r 120 -f /newton/raw "
  403. "tests/data/prep-20120323T1004")
  404. self.contain("error parsing input data")
  405. self.contain("can't parse value")
  406. # too few rows per line
  407. self.ok("create /insert/test float32_20")
  408. self.fail("insert -t -r 120 -f /insert/test "
  409. "tests/data/prep-20120323T1004")
  410. self.contain("error parsing input data")
  411. self.contain("wrong number of values")
  412. self.ok("destroy /insert/test")
  413. # empty data does nothing
  414. self.ok("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep "
  415. "/dev/null")
  416. # --quiet option
  417. self.ok("insert --quiet -t -r 120 -s @0 /newton/prep /dev/null")
  418. self.match("")
  419. # bad start time
  420. self.fail("insert -t -r 120 --start 'whatever' /newton/prep /dev/null")
  421. # Test negative times
  422. self.ok("insert --start @-10000000000 --end @1000000001 /newton/prep"
  423. " tests/data/timestamped")
  424. self.ok("extract -c /newton/prep --start min --end @1000000001")
  425. self.match("8\n")
  426. self.ok("remove /newton/prep --start min --end @1000000001")
  427. def test_07_detail_extended(self):
  428. # Just count the number of lines, it's probably fine
  429. self.ok("list --detail")
  430. lines_(self.captured, 8)
  431. self.ok("list --detail *prep")
  432. lines_(self.captured, 4)
  433. self.ok("list --detail *prep --start='23 Mar 2012 10:02'")
  434. lines_(self.captured, 3)
  435. self.ok("list --detail *prep --start='23 Mar 2012 10:05'")
  436. lines_(self.captured, 2)
  437. self.ok("list --detail *prep --start='23 Mar 2012 10:05:15'")
  438. lines_(self.captured, 2)
  439. self.contain("10:05:15.000")
  440. self.ok("list --detail *prep --start='23 Mar 2012 10:05:15.50'")
  441. lines_(self.captured, 2)
  442. self.contain("10:05:15.500")
  443. self.ok("list --detail *prep --start='23 Mar 2012 19:05:15.50'")
  444. lines_(self.captured, 2)
  445. self.contain("no intervals")
  446. self.ok("list --detail *prep --start='23 Mar 2012 10:05:15.50'"
  447. + " --end='23 Mar 2012 10:05:15.51'")
  448. lines_(self.captured, 2)
  449. self.contain("10:05:15.500")
  450. self.ok("list --detail")
  451. lines_(self.captured, 8)
  452. # Verify the "raw timestamp" output
  453. self.ok("list --detail *prep --timestamp-raw "
  454. "--start='23 Mar 2012 10:05:15.50'")
  455. lines_(self.captured, 2)
  456. self.contain("[ 1332497115500000 -> 1332497160000000 ]")
  457. # bad time
  458. self.fail("list --detail *prep -T --start='9332497115.612'")
  459. # good time
  460. self.ok("list --detail *prep -T --start='1332497115.612'")
  461. lines_(self.captured, 2)
  462. self.contain("[ 1332497115612000 -> 1332497160000000 ]")
  463. # Check --ext output
  464. self.ok("list --ext")
  465. lines_(self.captured, 9)
  466. self.ok("list -E -T")
  467. c = self.contain
  468. c("\n interval extents: 1332496800000000 -> 1332497160000000\n")
  469. c("\n total data: 43200 rows, 359.983336 seconds\n")
  470. c("\n interval extents: (no data)\n")
  471. c("\n total data: 0 rows, 0.000000 seconds\n")
  472. # Misc
  473. self.fail("list --ext --start='23 Mar 2012 10:05:15.50'")
  474. self.contain("--start and --end only make sense with --detail")
  475. def test_08_extract(self):
  476. # nonexistent stream
  477. self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")
  478. self.contain("error getting stream info")
  479. # reversed range
  480. self.fail("extract -a /newton/prep --start 2020-01-01 --end 2000-01-01")
  481. self.contain("start is after end")
  482. # empty ranges return error 2
  483. self.fail("extract -a /newton/prep " +
  484. "--start '23 Mar 2012 20:00:30' " +
  485. "--end '23 Mar 2012 20:00:31'",
  486. exitcode = 2, require_error = False)
  487. self.contain("no data")
  488. self.fail("extract -a /newton/prep " +
  489. "--start '23 Mar 2012 20:00:30.000001' " +
  490. "--end '23 Mar 2012 20:00:30.000002'",
  491. exitcode = 2, require_error = False)
  492. self.contain("no data")
  493. self.fail("extract -a /newton/prep " +
  494. "--start '23 Mar 2022 10:00:30' " +
  495. "--end '23 Mar 2022 10:00:31'",
  496. exitcode = 2, require_error = False)
  497. self.contain("no data")
  498. # unannotated empty extract is just empty, with an exit code of 2
  499. self.fail("extract /newton/prep " +
  500. "--start '23 Mar 2022 10:00:30' " +
  501. "--end '23 Mar 2022 10:00:31'",
  502. exitcode = 2, require_error = False)
  503. self.match("")
  504. # but are ok if we're just counting results
  505. self.ok("extract --count /newton/prep " +
  506. "--start '23 Mar 2012 20:00:30' " +
  507. "--end '23 Mar 2012 20:00:31'")
  508. self.match("0\n")
  509. self.ok("extract -c /newton/prep " +
  510. "--start '23 Mar 2012 20:00:30.000001' " +
  511. "--end '23 Mar 2012 20:00:30.000002'")
  512. self.match("0\n")
  513. # Extract needs --start and --end
  514. self.fail("extract -a /newton/prep")
  515. self.contain("arguments are required")
  516. self.fail("extract -a /newton/prep --start 2000-01-01")
  517. self.contain("arguments are required")
  518. self.fail("extract -a /newton/prep --end 2000-01-01")
  519. self.contain("arguments are required")
  520. # Check various dumps against stored copies of how they should appear
  521. def test(file, start, end, extra=""):
  522. self.ok("extract " + extra + " /newton/prep " +
  523. "--start '23 Mar 2012 " + start + "' " +
  524. "--end '23 Mar 2012 " + end + "'")
  525. self.matchfile("tests/data/extract-" + str(file))
  526. self.ok("extract --count " + extra + " /newton/prep " +
  527. "--start '23 Mar 2012 " + start + "' " +
  528. "--end '23 Mar 2012 " + end + "'")
  529. self.matchfilecount("tests/data/extract-" + str(file))
  530. test(1, "10:00:30", "10:00:31", extra="-a")
  531. test(1, "10:00:30.000000", "10:00:31", extra="-a")
  532. test(2, "10:00:30.000001", "10:00:31")
  533. test(2, "10:00:30.008333", "10:00:31")
  534. test(3, "10:00:30.008333", "10:00:30.008334")
  535. test(3, "10:00:30.008333", "10:00:30.016667")
  536. test(4, "10:00:30.008333", "10:00:30.025")
  537. test(5, "10:00:30", "10:00:31", extra="--annotate --bare")
  538. test(6, "10:00:30", "10:00:31", extra="-b")
  539. test(7, "10:00:30", "10:00:30.999", extra="-a -T")
  540. test(7, "10:00:30", "10:00:30.999", extra="-a --timestamp-raw")
  541. test(8, "10:01:59.9", "10:02:00.1", extra="--markup")
  542. test(8, "10:01:59.9", "10:02:00.1", extra="-m")
  543. # all data put in by tests
  544. self.ok("extract -a /newton/prep --start min --end max")
  545. lines_(self.captured, 43204)
  546. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  547. self.match("43200\n")
  548. # test binary mode
  549. self.fail("extract -c -B /newton/prep -s min -e max")
  550. self.contain("binary cannot be combined")
  551. self.fail("extract -m -B /newton/prep -s min -e max")
  552. self.contain("binary cannot be combined")
  553. self.ok("extract -B /newton/prep -s min -e max")
  554. eq_(len(self.captured_binary), 43200 * (8 + 8*4))
  555. # markup for 3 intervals, plus extra markup lines whenever we had
  556. # a "restart" from the nilmdb.stream_extract function
  557. self.ok("extract -m /newton/prep --start 2000-01-01 --end 2020-01-01")
  558. lines_(self.captured, 43210)
  559. def test_09_truncated(self):
  560. # Test truncated responses by overriding the nilmdb max_results
  561. server_stop()
  562. server_start(max_results = 2)
  563. self.ok("list --detail")
  564. lines_(self.captured, 8)
  565. server_stop()
  566. server_start()
  567. def test_10_remove(self):
  568. # Removing data
  569. # Try nonexistent stream
  570. self.fail("remove /no/such/foo --start 2000-01-01 --end 2020-01-01")
  571. self.contain("no stream matched path")
  572. # empty or backward ranges return errors
  573. self.fail("remove /newton/prep --start 2020-01-01 --end 2000-01-01")
  574. self.contain("start must precede end")
  575. self.fail("remove /newton/prep " +
  576. "--start '23 Mar 2012 10:00:30' " +
  577. "--end '23 Mar 2012 10:00:30'")
  578. self.contain("start must precede end")
  579. self.fail("remove /newton/prep " +
  580. "--start '23 Mar 2012 10:00:30.000001' " +
  581. "--end '23 Mar 2012 10:00:30.000001'")
  582. self.contain("start must precede end")
  583. self.fail("remove /newton/prep " +
  584. "--start '23 Mar 2022 10:00:30' " +
  585. "--end '23 Mar 2022 10:00:30'")
  586. self.contain("start must precede end")
  587. # Verbose
  588. self.ok("remove -c /newton/prep " +
  589. "--start '23 Mar 2022 20:00:30' " +
  590. "--end '23 Mar 2022 20:00:31'")
  591. self.match("0\n")
  592. self.ok("remove --count /newton/prep " +
  593. "--start '23 Mar 2022 20:00:30' " +
  594. "--end '23 Mar 2022 20:00:31'")
  595. self.match("0\n")
  596. self.ok("remove -c /newton/prep /newton/pre* " +
  597. "--start '23 Mar 2022 20:00:30' " +
  598. "--end '23 Mar 2022 20:00:31'")
  599. self.match("Removing from /newton/prep\n0\n" +
  600. "Removing from /newton/prep\n0\n")
  601. # Make sure we have the data we expect
  602. self.ok("list -l --detail /newton/prep")
  603. self.match("/newton/prep float32_8\n" +
  604. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  605. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  606. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  607. " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n"
  608. " [ Fri, 23 Mar 2012 10:04:00.000000 +0000"
  609. " -> Fri, 23 Mar 2012 10:06:00.000000 +0000 ]\n")
  610. # Remove various chunks of prep data and make sure
  611. # they're gone.
  612. self.ok("remove -c /newton/prep " +
  613. "--start '23 Mar 2012 10:00:30' " +
  614. "--end '23 Mar 2012 10:00:40'")
  615. self.match("1200\n")
  616. self.ok("remove -c /newton/prep " +
  617. "--start '23 Mar 2012 10:00:10' " +
  618. "--end '23 Mar 2012 10:00:20'")
  619. self.match("1200\n")
  620. self.ok("remove -c /newton/prep " +
  621. "--start '23 Mar 2012 10:00:05' " +
  622. "--end '23 Mar 2012 10:00:25'")
  623. self.match("1200\n")
  624. self.ok("remove -c /newton/prep " +
  625. "--start '23 Mar 2012 10:03:50' " +
  626. "--end '23 Mar 2012 10:06:50'")
  627. self.match("15600\n")
  628. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  629. self.match("24000\n")
  630. # See the missing chunks in list output
  631. self.ok("list --layout --detail /newton/prep")
  632. self.match("/newton/prep float32_8\n" +
  633. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  634. " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
  635. " [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
  636. " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
  637. " [ Fri, 23 Mar 2012 10:00:40.000000 +0000"
  638. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  639. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  640. " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
  641. # Remove all data, verify it's missing
  642. self.ok("remove /newton/prep --start 2000-01-01 --end 2020-01-01")
  643. self.match("") # no count requested this time
  644. self.ok("list -l --detail /newton/prep")
  645. self.match("/newton/prep float32_8\n" +
  646. " (no intervals)\n")
  647. # Reinsert some data, to verify that no overlaps with deleted
  648. # data are reported
  649. for minute in ["0", "2"]:
  650. self.ok("insert --timestamp -f --rate 120 /newton/prep"
  651. " tests/data/prep-20120323T100" + minute)
  652. def test_11_destroy(self):
  653. # Delete records
  654. self.ok("destroy --help")
  655. self.fail("destroy")
  656. self.contain("the following arguments are required")
  657. self.fail("destroy /no/such/stream")
  658. self.contain("no stream matched path")
  659. self.fail("destroy -R /no/such/stream")
  660. self.contain("no stream matched path")
  661. self.fail("destroy asdfasdf")
  662. self.contain("no stream matched path")
  663. # From previous tests, we have:
  664. self.ok("list -l")
  665. self.match("/newton/prep float32_8\n"
  666. "/newton/raw uint16_6\n"
  667. "/newton/zzz/rawnotch uint16_9\n")
  668. # Notice how they're not empty
  669. self.ok("list --detail")
  670. lines_(self.captured, 7)
  671. # Fail to destroy because intervals still present
  672. self.fail("destroy /newton/prep")
  673. self.contain("all intervals must be removed")
  674. self.ok("list --detail")
  675. lines_(self.captured, 7)
  676. # Destroy for real
  677. self.ok("destroy -R /n*/prep")
  678. self.ok("list -l")
  679. self.match("/newton/raw uint16_6\n"
  680. "/newton/zzz/rawnotch uint16_9\n")
  681. self.ok("destroy /newton/zzz/rawnotch")
  682. self.ok("list -l")
  683. self.match("/newton/raw uint16_6\n")
  684. self.ok("destroy /newton/raw")
  685. self.ok("create /newton/raw uint16_6")
  686. # Specify --remove with no data
  687. self.ok("destroy --remove /newton/raw")
  688. self.ok("list")
  689. self.match("")
  690. # Re-create a previously deleted location, and some new ones
  691. rebuild = [ "/newton/prep", "/newton/zzz",
  692. "/newton/raw", "/newton/asdf/qwer" ]
  693. for path in rebuild:
  694. # Create the path
  695. self.ok("create " + path + " float32_8")
  696. self.ok("list")
  697. self.contain(path)
  698. # Make sure it was created empty
  699. self.ok("list --detail " + path)
  700. self.contain("(no intervals)")
  701. def test_12_unicode(self):
  702. # Unicode paths.
  703. self.ok("destroy /newton/asdf/qwer")
  704. self.ok("destroy /newton/prep /newton/raw")
  705. self.ok("destroy /newton/zzz")
  706. self.ok("create /dรผsseldorf/raw uint16_6")
  707. self.ok("list -l --detail")
  708. self.contain("/dรผsseldorf/raw uint16_6")
  709. self.contain("(no intervals)")
  710. # Unicode metadata
  711. self.ok("metadata /dรผsseldorf/raw --set ฮฑ=beta 'ฮณ=ฮด'")
  712. self.ok("metadata /dรผsseldorf/raw --update 'ฮฑ=ฮฒ ฮต ฯ„ ฮฑ'")
  713. self.ok("metadata /dรผsseldorf/raw")
  714. self.match("ฮฑ=ฮฒ ฮต ฯ„ ฮฑ\nฮณ=ฮด\n")
  715. self.ok("destroy /dรผsseldorf/raw")
  716. def test_13_files(self):
  717. # Test BulkData's ability to split into multiple files,
  718. # by forcing the file size to be really small.
  719. # Also increase the initial nrows, so that start/end positions
  720. # in the database are very large (> 32 bit)
  721. server_stop()
  722. server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
  723. "files_per_dir" : 3,
  724. "initial_nrows" : 2**40 })
  725. # Fill data
  726. self.ok("create /newton/prep float32_8")
  727. os.environ['TZ'] = "UTC"
  728. with open("tests/data/prep-20120323T1004-timestamped") as input:
  729. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  730. input)
  731. # Extract it
  732. self.ok("extract /newton/prep --start '2000-01-01' " +
  733. "--end '2012-03-23 10:04:01'")
  734. lines_(self.captured, 120)
  735. self.ok("extract /newton/prep --start '2000-01-01' " +
  736. "--end '2022-03-23 10:04:01'")
  737. lines_(self.captured, 14400)
  738. # Make sure there were lots of files generated in the database
  739. # dir
  740. nfiles = 0
  741. for (dirpath, dirnames, filenames) in os.walk(testdb):
  742. nfiles += len(filenames)
  743. assert(nfiles > 500)
  744. # Make sure we can restart the server with a different file
  745. # size and have it still work
  746. server_stop()
  747. server_start()
  748. self.ok("extract /newton/prep --start '2000-01-01' " +
  749. "--end '2022-03-23 10:04:01'")
  750. lines_(self.captured, 14400)
  751. # Now recreate the data one more time and make sure there are
  752. # fewer files.
  753. self.ok("destroy --remove /newton/prep")
  754. self.fail("destroy /newton/prep") # already destroyed
  755. self.ok("create /newton/prep float32_8")
  756. os.environ['TZ'] = "UTC"
  757. with open("tests/data/prep-20120323T1004-timestamped") as input:
  758. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  759. input)
  760. nfiles = 0
  761. for (dirpath, dirnames, filenames) in os.walk(testdb):
  762. nfiles += len(filenames)
  763. lt_(nfiles, 50)
  764. self.ok("destroy -R /newton/prep") # destroy again
  765. def test_14_remove_files(self):
  766. # Limit max_removals, to cover more functionality.
  767. server_stop()
  768. server_start(max_removals = 4321,
  769. bulkdata_args = { "file_size" : 920, # 23 rows per file
  770. "files_per_dir" : 3,
  771. "initial_nrows" : 2**40 })
  772. self.do_remove_files()
  773. self.ok("destroy -R /newton/prep") # destroy again
  774. def test_14b_remove_files_maxint(self):
  775. # Limit max_int_removals, to cover more functionality.
  776. server_stop()
  777. server_start(max_int_removals = 1,
  778. bulkdata_args = { "file_size" : 920, # 23 rows per file
  779. "files_per_dir" : 3,
  780. "initial_nrows" : 2**40 })
  781. self.do_remove_files()
  782. def do_remove_files(self):
  783. # Test BulkData's ability to remove when data is split into
  784. # multiple files. Should be a fairly comprehensive test of
  785. # remove functionality.
  786. # Insert data. Just for fun, insert out of order
  787. self.ok("create /newton/prep float32_8")
  788. os.environ['TZ'] = "UTC"
  789. self.ok("insert -t --filename --rate 120 /newton/prep "
  790. "tests/data/prep-20120323T1002")
  791. self.ok("insert -t --filename --rate 120 /newton/prep "
  792. "tests/data/prep-20120323T1000")
  793. # Should take up about 2.8 MB here (including directory entries)
  794. du_before = nilmdb.utils.diskusage.du(testdb)
  795. # Make sure we have the data we expect
  796. self.ok("list -l --detail")
  797. self.match("/newton/prep float32_8\n" +
  798. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  799. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  800. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  801. " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n")
  802. # Remove various chunks of prep data and make sure
  803. # they're gone.
  804. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  805. self.match("28800\n")
  806. self.ok("remove -c /newton/prep " +
  807. "--start '23 Mar 2012 10:00:30' " +
  808. "--end '23 Mar 2012 10:03:30'")
  809. self.match("21600\n")
  810. self.ok("remove -c /newton/prep " +
  811. "--start '23 Mar 2012 10:00:10' " +
  812. "--end '23 Mar 2012 10:00:20'")
  813. self.match("1200\n")
  814. self.ok("remove -c /newton/prep " +
  815. "--start '23 Mar 2012 10:00:05' " +
  816. "--end '23 Mar 2012 10:00:25'")
  817. self.match("1200\n")
  818. self.ok("remove -c /newton/prep " +
  819. "--start '23 Mar 2012 10:03:50' " +
  820. "--end '23 Mar 2012 10:06:50'")
  821. self.match("1200\n")
  822. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  823. self.match("3600\n")
  824. # See the missing chunks in list output
  825. self.ok("list -l --detail")
  826. self.match("/newton/prep float32_8\n" +
  827. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  828. " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
  829. " [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
  830. " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
  831. " [ Fri, 23 Mar 2012 10:03:30.000000 +0000"
  832. " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
  833. # We have 1/8 of the data that we had before, so the file size
  834. # should have dropped below 1/4 of what it used to be
  835. du_after = nilmdb.utils.diskusage.du(testdb)
  836. lt_(du_after, (du_before / 4))
  837. # Remove anything that came from the 10:02 data file
  838. self.ok("remove /newton/prep " +
  839. "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
  840. # Re-insert 19 lines from that file, then remove them again.
  841. # With the specific file_size above, this will cause the last
  842. # file in the bulk data storage to be exactly file_size large,
  843. # so removing the data should also remove that last file.
  844. self.ok("insert --timestamp -f --rate 120 /newton/prep " +
  845. "tests/data/prep-20120323T1002-first19lines")
  846. self.ok("remove /newton/prep " +
  847. "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
  848. # Shut down and restart server, to force nrows to get refreshed.
  849. server_stop()
  850. server_start()
  851. # Re-add the full 10:02 data file. This tests adding new data once
  852. # we removed data near the end.
  853. self.ok("insert -t -f -r 120 /newton/prep "
  854. "tests/data/prep-20120323T1002")
  855. # See if we can extract it all
  856. self.ok("extract /newton/prep --start 2000-01-01 --end 2020-01-01")
  857. lines_(self.captured, 15600)
  858. def test_15_intervals_diff(self):
  859. # Test "intervals" and "intervals --diff" command.
  860. os.environ['TZ'] = "UTC"
  861. self.ok("create /diff/1 uint8_1")
  862. self.match("")
  863. self.ok("intervals /diff/1")
  864. self.match("")
  865. self.ok("intervals /diff/1 --diff /diff/1")
  866. self.match("")
  867. self.ok("intervals --diff /diff/1 /diff/1")
  868. self.match("")
  869. self.fail("intervals /diff/2")
  870. self.fail("intervals /diff/1 -d /diff/2")
  871. self.ok("create /diff/2 uint8_1")
  872. self.ok("intervals -T /diff/1 -d /diff/2")
  873. self.match("")
  874. self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/1 /dev/null")
  875. self.ok("intervals /diff/1")
  876. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  877. "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
  878. self.ok("intervals /diff/1 -d /diff/2")
  879. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  880. "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
  881. self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/2 /dev/null")
  882. self.ok("intervals /diff/1 -d /diff/2")
  883. self.match("")
  884. self.ok("insert -s 01-01-2001 -e 01-01-2002 /diff/1 /dev/null")
  885. self.ok("insert -s 01-01-2002 -e 01-01-2003 /diff/2 /dev/null")
  886. self.ok("intervals /diff/1 -d /diff/2")
  887. self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
  888. "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n")
  889. self.ok("insert -s 01-01-2004 -e 01-01-2005 /diff/1 /dev/null")
  890. self.ok("intervals /diff/1 -d /diff/2")
  891. self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
  892. "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n"
  893. "[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
  894. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  895. self.fail("intervals -s 01-01-2003 -e 01-01-2000 /diff/1 -d /diff/2")
  896. self.ok("intervals -s 01-01-2003 -e 01-01-2008 /diff/1 -d /diff/2")
  897. self.match("[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
  898. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  899. # optimize
  900. self.ok("insert -s 01-01-2002 -e 01-01-2004 /diff/1 /dev/null")
  901. self.ok("intervals /diff/1")
  902. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  903. "> Thu, 01 Jan 2004 00:00:00.000000 +0000 ]\n"
  904. "[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
  905. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  906. self.ok("intervals /diff/1 --optimize")
  907. self.ok("intervals /diff/1 -o")
  908. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  909. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  910. self.ok("destroy -R /diff/1")
  911. self.ok("destroy -R /diff/2")
  912. def test_16_rename(self):
  913. # Test renaming. Force file size smaller so we get more files
  914. server_stop()
  915. recursive_unlink(testdb)
  916. server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
  917. "files_per_dir" : 3 })
  918. # Fill data
  919. self.ok("create /newton/prep float32_8")
  920. os.environ['TZ'] = "UTC"
  921. with open("tests/data/prep-20120323T1004-timestamped") as input:
  922. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  923. input)
  924. # Extract it
  925. self.ok("extract /newton/prep --start '2000-01-01' " +
  926. "--end '2012-03-23 10:04:01'")
  927. extract_before = self.captured
  928. def check_path(*components):
  929. # Verify the paths look right on disk
  930. seek = os.path.join(testdb, "data", *components)
  931. for (dirpath, dirnames, filenames) in os.walk(testdb):
  932. if "_format" in filenames:
  933. if dirpath == seek:
  934. break
  935. raise AssertionError("data also found at " + dirpath)
  936. else:
  937. raise AssertionError("data not found at " + seek)
  938. # Verify "list" output
  939. self.ok("list -l")
  940. self.match("/" + "/".join(components) + " float32_8\n")
  941. # Lots of renames
  942. check_path("newton", "prep")
  943. self.fail("rename /newton/prep /newton/prep")
  944. self.contain("old and new paths are the same")
  945. check_path("newton", "prep")
  946. self.fail("rename /newton/prep /newton")
  947. self.contain("path must contain at least one folder")
  948. self.fail("rename /newton/prep /newton/prep/")
  949. self.contain("invalid path")
  950. self.ok("rename /newton/prep /newton/foo/1")
  951. check_path("newton", "foo", "1")
  952. self.ok("rename /newton/foo/1 /newton/foo")
  953. check_path("newton", "foo")
  954. self.ok("rename /newton/foo /totally/different/thing")
  955. check_path("totally", "different", "thing")
  956. self.ok("rename /totally/different/thing /totally/something")
  957. check_path("totally", "something")
  958. self.ok("rename /totally/something /totally/something/cool")
  959. check_path("totally", "something", "cool")
  960. self.ok("rename /totally/something/cool /foo/bar")
  961. check_path("foo", "bar")
  962. self.ok("create /xxx/yyy/zzz float32_8")
  963. self.fail("rename /foo/bar /xxx/yyy")
  964. self.contain("subdirs of this path already exist")
  965. self.fail("rename /foo/bar /xxx/yyy/zzz")
  966. self.contain("stream already exists at this path")
  967. self.fail("rename /foo/bar /xxx/yyy/zzz/www")
  968. self.contain("path is subdir of existing node")
  969. self.ok("rename /foo/bar /xxx/yyy/mmm")
  970. self.ok("destroy -R /xxx/yyy/zzz")
  971. check_path("xxx", "yyy", "mmm")
  972. # Extract it at the final path
  973. self.ok("extract /xxx/yyy/mmm --start '2000-01-01' " +
  974. "--end '2012-03-23 10:04:01'")
  975. eq_(self.captured, extract_before)
  976. self.ok("destroy -R /xxx/yyy/mmm")
  977. # Make sure temporary rename dirs weren't left around
  978. for (dirpath, dirnames, filenames) in os.walk(testdb):
  979. if "rename-" in dirpath:
  980. raise AssertionError("temporary directories not cleaned up")
  981. if "totally" in dirpath or "newton" in dirpath:
  982. raise AssertionError("old directories not cleaned up")
  983. server_stop()
  984. server_start()
  985. def test_05b_completion(self):
  986. # Test bash completion. This depends on some data put in the DB by
  987. # earlier tests, so the execution order is important.
  988. def complete(line, expect="<unspecified>"):
  989. # set env vars
  990. env = {
  991. '_ARGCOMPLETE': '1',
  992. 'COMP_LINE': line,
  993. 'COMP_POINT': str(len(line)),
  994. 'COMP_TYPE': '8',
  995. 'NILMDB_URL': "http://localhost:32180/",
  996. }
  997. for (k, v) in env.items():
  998. os.environ[k] = v
  999. # create pipe for completion output
  1000. output = io.BytesIO()
  1001. # ensure argcomplete won't mess with any FDs
  1002. def fake_fdopen(fd, mode):
  1003. return io.BytesIO()
  1004. old_fdopen = os.fdopen
  1005. os.fdopen = fake_fdopen
  1006. # run cli
  1007. cmdline = nilmdb.cmdline.Cmdline([])
  1008. cmdline.complete_output_stream = output
  1009. try:
  1010. cmdline.run()
  1011. sys.exit(0)
  1012. except SystemExit as e:
  1013. exitcode = e.code
  1014. eq_(exitcode, 0)
  1015. # clean up
  1016. os.fdopen = old_fdopen
  1017. for (k, v) in env.items():
  1018. del os.environ[k]
  1019. # read completion output
  1020. comp = output.getvalue()
  1021. # replace completion separators with commas, for clarity
  1022. cleaned = comp.replace(b'\x0b', b',').decode('utf-8')
  1023. # expect the given match or prefix
  1024. if expect.endswith('*'):
  1025. if not cleaned.startswith(expect[:-1]):
  1026. raise AssertionError(("completions:\n '%s'\n"
  1027. "don't start with:\n '%s'") %
  1028. (cleaned, expect[:-1]))
  1029. else:
  1030. if cleaned != expect:
  1031. raise AssertionError(("completions:\n '%s'\n"
  1032. "don't match:\n '%s'") %
  1033. (cleaned, expect))
  1034. complete("nilmtool -u ", "")
  1035. complete("nilmtool list ", "-h,--help,-E,--ext*")
  1036. complete("nilmtool list --st", "--start ")
  1037. complete("nilmtool list --start ", "")
  1038. complete("nilmtool list /", "/newton/prep,/newton/raw*")
  1039. complete("nilmtool create /foo int3", "int32_1,int32_2*")
  1040. complete("nilmtool metadata /newton/raw --get a",
  1041. "a_๐“ด๐“ฎ๐”‚,a_key,a_๐—ธ๐—ฒ๐˜†")
  1042. complete("nilmtool metadata /newton/raw --set a",
  1043. "a_๐“ด๐“ฎ๐”‚=value,a_key=๐“ฟ๐“ช๐“ต๐“พ๐“ฎ,a_๐—ธ๐—ฒ๐˜†=๐˜ƒ๐—ฎ๐—น๐˜‚๐—ฒ")
  1044. complete("nilmtool metadata /newton/raw --set a_๐—ธ", "a_๐—ธ๐—ฒ๐˜†=๐˜ƒ๐—ฎ๐—น๐˜‚๐—ฒ ")
  1045. complete("nilmtool metadata '' --set a", "")
  1046. self.run("list")