You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

1142 lines
44 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb.server
  3. from nilmdb.utils.printf import *
  4. import nilmdb.cmdline
  5. import datetime_tz
  6. import unittest
  7. from nose.tools import *
  8. from nose.tools import assert_raises
  9. import itertools
  10. import os
  11. import re
  12. import sys
  13. import io
  14. import shlex
  15. import warnings
  16. from testutil.helpers import *
  17. testdb = "tests/cmdline-testdb"
  18. def server_start(max_results = None,
  19. max_removals = None,
  20. max_int_removals = None,
  21. bulkdata_args = {}):
  22. global test_server, test_db
  23. # Start web app on a custom port
  24. test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(
  25. testdb,
  26. max_results = max_results,
  27. max_removals = max_removals,
  28. max_int_removals = max_int_removals,
  29. bulkdata_args = bulkdata_args)
  30. test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
  31. port = 32180, stoppable = False,
  32. fast_shutdown = True,
  33. force_traceback = False)
  34. test_server.start(blocking = False)
  35. def server_stop():
  36. global test_server, test_db
  37. # Close web app
  38. test_server.stop()
  39. test_db.close()
  40. def setup_module():
  41. global test_server, test_db
  42. # Clear out DB
  43. recursive_unlink(testdb)
  44. server_start()
  45. def teardown_module():
  46. server_stop()
  47. class TestCmdline(object):
  48. def run(self, arg_string, infile=None, outfile=None):
  49. """Run a cmdline client with the specified argument string,
  50. passing the given input. Save the output and exit code."""
  51. # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
  52. os.environ['NILMDB_URL'] = "http://localhost:32180/"
  53. class stdio_wrapper:
  54. def __init__(self, stdin, stdout, stderr):
  55. self.io = (stdin, stdout, stderr)
  56. def __enter__(self):
  57. self.saved = ( sys.stdin, sys.stdout, sys.stderr )
  58. ( sys.stdin, sys.stdout, sys.stderr ) = self.io
  59. def __exit__(self, type, value, traceback):
  60. ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
  61. # Empty input if none provided
  62. if infile is None:
  63. infile = io.StringIO("")
  64. # Capture stderr
  65. errfile = io.StringIO()
  66. if outfile is None:
  67. # If no output file, capture stdout with stderr
  68. outfile = errfile
  69. with stdio_wrapper(infile, outfile, errfile) as s:
  70. try:
  71. args = shlex.split(arg_string)
  72. nilmdb.cmdline.Cmdline(args).run()
  73. sys.exit(0)
  74. except SystemExit as e:
  75. exitcode = e.code
  76. captured = outfile.getvalue()
  77. self.captured = captured
  78. self.exitcode = exitcode
  79. def ok(self, arg_string, infile = None):
  80. self.run(arg_string, infile)
  81. if self.exitcode != 0:
  82. self.dump()
  83. eq_(self.exitcode, 0)
  84. def fail(self, arg_string, infile = None,
  85. exitcode = None, require_error = True):
  86. self.run(arg_string, infile)
  87. if exitcode is not None and self.exitcode != exitcode:
  88. # Wrong exit code
  89. self.dump()
  90. eq_(self.exitcode, exitcode)
  91. if self.exitcode == 0:
  92. # Success, when we wanted failure
  93. self.dump()
  94. ne_(self.exitcode, 0)
  95. # Make sure the output contains the word "error" at the
  96. # beginning of a line, but only if an exitcode wasn't
  97. # specified.
  98. if require_error and not re.search("^error",
  99. self.captured, re.MULTILINE):
  100. raise AssertionError("command failed, but output doesn't "
  101. "contain the string 'error'")
  102. def contain(self, checkstring):
  103. in_(checkstring, self.captured)
  104. def match(self, checkstring):
  105. eq_(checkstring, self.captured)
  106. def matchfile(self, file):
  107. # Captured data should match file contents exactly
  108. with open(file) as f:
  109. contents = f.read()
  110. if contents != self.captured:
  111. print("--- reference file (first 1000 bytes):\n")
  112. print(contents[0:1000] + "\n")
  113. print("--- captured data (first 1000 bytes):\n")
  114. print(self.captured[0:1000] + "\n")
  115. zipped = itertools.zip_longest(contents, self.captured)
  116. for (n, (a, b)) in enumerate(zipped):
  117. if a != b:
  118. print("--- first difference is at offset", n)
  119. print("--- reference:", repr(a))
  120. print("--- captured:", repr(b))
  121. break
  122. raise AssertionError("captured data doesn't match " + file)
  123. def matchfilecount(self, file):
  124. # Last line of captured data should match the number of
  125. # non-commented lines in file
  126. count = 0
  127. with open(file) as f:
  128. for line in f:
  129. if line[0] != '#':
  130. count += 1
  131. eq_(self.captured.splitlines()[-1], sprintf("%d", count))
  132. def dump(self):
  133. printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
  134. def test_01_basic(self):
  135. # help
  136. self.ok("--help")
  137. self.contain("usage:")
  138. # help
  139. self.ok("--version")
  140. ver = self.captured
  141. self.ok("list --version")
  142. eq_(self.captured, ver)
  143. # fail for no args
  144. self.fail("")
  145. # fail for no such option
  146. self.fail("--nosuchoption")
  147. # fail for bad command
  148. self.fail("badcommand")
  149. # try some URL constructions
  150. self.fail("--url http://nosuchurl/ info")
  151. self.contain("error connecting to server")
  152. self.fail("--url nosuchurl info")
  153. self.contain("error connecting to server")
  154. self.fail("-u nosuchurl/foo info")
  155. self.contain("error connecting to server")
  156. self.fail("-u localhost:1 info")
  157. self.contain("error connecting to server")
  158. self.ok("-u localhost:32180 info")
  159. self.ok("info")
  160. # Duplicated arguments should fail, but this isn't implemented
  161. # due to it being kind of a pain with argparse.
  162. if 0:
  163. self.fail("-u url1 -u url2 info")
  164. self.contain("duplicated argument")
  165. self.fail("list --detail --detail")
  166. self.contain("duplicated argument")
  167. self.fail("list --detail --path path1 --path path2")
  168. self.contain("duplicated argument")
  169. self.fail("extract --start 2000-01-01 --start 2001-01-02")
  170. self.contain("duplicated argument")
  171. # Verify that "help command" and "command --help" are identical
  172. # for all commands.
  173. self.fail("")
  174. m = re.search(r"{(.*)}", self.captured)
  175. for command in [""] + m.group(1).split(','):
  176. self.ok(command + " --help")
  177. cap1 = self.captured
  178. self.ok("help " + command)
  179. cap2 = self.captured
  180. self.ok("help " + command + " asdf --url --zxcv -")
  181. cap3 = self.captured
  182. eq_(cap1, cap2)
  183. eq_(cap2, cap3)
  184. def test_02_parsetime(self):
  185. os.environ['TZ'] = "America/New_York"
  186. test = datetime_tz.datetime_tz.now()
  187. u2ts = nilmdb.utils.time.unix_to_timestamp
  188. parse_time = nilmdb.utils.time.parse_time
  189. eq_(parse_time(str(test)), u2ts(test.totimestamp()))
  190. test = u2ts(datetime_tz.datetime_tz.smartparse("20120405 1400-0400").
  191. totimestamp())
  192. eq_(parse_time("hi there 20120405 1400-0400 testing! 123"), test)
  193. eq_(parse_time("20120405 1800 UTC"), test)
  194. eq_(parse_time("20120405 1400-0400 UTC"), test)
  195. for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
  196. with assert_raises(ValueError):
  197. x = parse_time(badtime)
  198. x = parse_time("now")
  199. eq_(parse_time("snapshot-20120405-140000.raw.gz"), test)
  200. eq_(parse_time("prep-20120405T1400"), test)
  201. eq_(parse_time("1333648800.0"), test)
  202. eq_(parse_time("1333648800000000"), test)
  203. eq_(parse_time("@1333648800000000"), test)
  204. eq_(parse_time("min"), nilmdb.utils.time.min_timestamp)
  205. eq_(parse_time("max"), nilmdb.utils.time.max_timestamp)
  206. with assert_raises(ValueError):
  207. parse_time("@hashtag12345")
  208. def test_03_info(self):
  209. self.ok("info")
  210. self.contain("Server URL: http://localhost:32180/")
  211. self.contain("Client version: " + nilmdb.__version__)
  212. self.contain("Server version: " + test_server.version)
  213. self.contain("Server database path")
  214. self.contain("Server disk space used by NilmDB")
  215. self.contain("Server disk space used by other")
  216. self.contain("Server disk space reserved")
  217. self.contain("Server disk space free")
  218. def test_04_createlist(self):
  219. # Basic stream tests, like those in test_client.
  220. # No streams
  221. self.ok("list")
  222. self.match("")
  223. # Bad paths
  224. self.fail("create foo/bar/baz float32_8")
  225. self.contain("paths must start with /")
  226. self.fail("create /foo float32_8")
  227. self.contain("invalid path")
  228. self.fail("create /newton/prep/ float32_8")
  229. self.contain("invalid path")
  230. self.fail("create /newton/_format/prep float32_8")
  231. self.contain("path name is invalid")
  232. self.fail("create /_format/newton/prep float32_8")
  233. self.contain("path name is invalid")
  234. self.fail("create /newton/prep/_format float32_8")
  235. self.contain("path name is invalid")
  236. # Bad layout type
  237. self.fail("create /newton/prep NoSuchLayout")
  238. self.contain("no such layout")
  239. self.fail("create /newton/prep float32_0")
  240. self.contain("no such layout")
  241. self.fail("create /newton/prep float33_1")
  242. self.contain("no such layout")
  243. # Create a few streams
  244. self.ok("create /newton/zzz/rawnotch uint16_9")
  245. self.ok("create /newton/prep float32_8")
  246. self.ok("create /newton/raw uint16_6")
  247. self.ok("create /newton/raw~decim-1234 uint16_6")
  248. # Create a stream that already exists
  249. self.fail("create /newton/raw uint16_6")
  250. self.contain("stream already exists at this path")
  251. # Should not be able to create a stream with another stream as
  252. # its parent
  253. self.fail("create /newton/prep/blah float32_8")
  254. self.contain("path is subdir of existing node")
  255. # Should not be able to create a stream at a location that
  256. # has other nodes as children
  257. self.fail("create /newton/zzz float32_8")
  258. self.contain("subdirs of this path already exist")
  259. # Verify we got those 4 streams and they're returned in
  260. # alphabetical order.
  261. self.ok("list -l")
  262. self.match("/newton/prep float32_8\n"
  263. "/newton/raw uint16_6\n"
  264. "/newton/raw~decim-1234 uint16_6\n"
  265. "/newton/zzz/rawnotch uint16_9\n")
  266. # No decimated streams if -n specified
  267. self.ok("list -n -l")
  268. self.match("/newton/prep float32_8\n"
  269. "/newton/raw uint16_6\n"
  270. "/newton/zzz/rawnotch uint16_9\n")
  271. # Delete that decimated stream
  272. self.ok("destroy /newton/raw~decim-1234")
  273. # Match just one type or one path. Also check
  274. # that --path is optional
  275. self.ok("list --layout /newton/raw")
  276. self.match("/newton/raw uint16_6\n")
  277. # Wildcard matches
  278. self.ok("list *zzz*")
  279. self.match("/newton/zzz/rawnotch\n")
  280. # reversed range
  281. self.fail("list /newton/prep --start 2020-01-01 --end 2000-01-01")
  282. self.contain("start must precede end")
  283. def test_05_metadata(self):
  284. # Set / get metadata
  285. self.fail("metadata")
  286. self.fail("metadata --get")
  287. self.ok("metadata /newton/prep")
  288. self.match("")
  289. self.ok("metadata /newton/raw --get")
  290. self.match("")
  291. self.ok("metadata /newton/prep --set "
  292. "'description=The Data' "
  293. "v_scale=1.234")
  294. self.ok("metadata /newton/raw --update "
  295. "'description=The Data'")
  296. self.ok("metadata /newton/raw --update "
  297. "v_scale=1.234")
  298. # various parsing tests
  299. self.ok("metadata /newton/raw --update foo=")
  300. self.fail("metadata /newton/raw --update =bar")
  301. self.fail("metadata /newton/raw --update foo==bar")
  302. self.fail("metadata /newton/raw --update foo;bar")
  303. # errors
  304. self.fail("metadata /newton/nosuchstream foo=bar")
  305. self.contain("unrecognized arguments")
  306. self.fail("metadata /newton/nosuchstream")
  307. self.contain("No stream at path")
  308. self.fail("metadata /newton/nosuchstream --set foo=bar")
  309. self.contain("No stream at path")
  310. self.fail("metadata /newton/nosuchstream --delete")
  311. self.contain("No stream at path")
  312. self.ok("metadata /newton/prep")
  313. self.match("description=The Data\nv_scale=1.234\n")
  314. self.ok("metadata /newton/prep --get")
  315. self.match("description=The Data\nv_scale=1.234\n")
  316. self.ok("metadata /newton/prep --get descr")
  317. self.match("descr=\n")
  318. self.ok("metadata /newton/prep --get description")
  319. self.match("description=The Data\n")
  320. self.ok("metadata /newton/prep --get description v_scale")
  321. self.match("description=The Data\nv_scale=1.234\n")
  322. self.ok("metadata /newton/prep --set "
  323. "'description=The Data'")
  324. self.ok("metadata /newton/prep --get")
  325. self.match("description=The Data\n")
  326. self.fail("metadata /newton/nosuchpath")
  327. self.contain("No stream at path /newton/nosuchpath")
  328. self.ok("metadata /newton/prep --delete")
  329. self.ok("metadata /newton/prep --get")
  330. self.match("")
  331. self.ok("metadata /newton/prep --set "
  332. "'description=The Data' "
  333. "v_scale=1.234")
  334. self.ok("metadata /newton/prep --delete v_scale")
  335. self.ok("metadata /newton/prep --get")
  336. self.match("description=The Data\n")
  337. self.ok("metadata /newton/prep --set description=")
  338. self.ok("metadata /newton/prep --get")
  339. self.match("")
  340. def test_06_insert(self):
  341. self.ok("insert --help")
  342. self.fail("insert -s 2000 -e 2001 /foo/bar baz")
  343. self.contain("error getting stream info")
  344. self.fail("insert -s 2000 -e 2001 /newton/prep baz")
  345. self.match("error opening input file baz\n")
  346. self.fail("insert /newton/prep --timestamp -f -r 120")
  347. self.contain("error extracting start time")
  348. self.fail("insert /newton/prep --timestamp -r 120")
  349. self.contain("need --start or --filename")
  350. self.fail("insert /newton/prep "
  351. "tests/data/prep-20120323T1000")
  352. # insert pre-timestamped data, with bad times (non-monotonic)
  353. os.environ['TZ'] = "UTC"
  354. with open("tests/data/prep-20120323T1004-badtimes") as input:
  355. self.fail("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  356. input)
  357. self.contain("error parsing input data")
  358. self.contain("line 7")
  359. self.contain("timestamp is not monotonically increasing")
  360. # insert pre-timestamped data, from stdin
  361. os.environ['TZ'] = "UTC"
  362. with open("tests/data/prep-20120323T1004-timestamped") as input:
  363. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  364. input)
  365. # insert data with normal timestamper from filename
  366. os.environ['TZ'] = "UTC"
  367. self.ok("insert --timestamp -f --rate 120 /newton/prep "
  368. "tests/data/prep-20120323T1000")
  369. self.fail("insert -t --filename /newton/prep "
  370. "tests/data/prep-20120323T1002")
  371. self.contain("rate is needed")
  372. self.ok("insert -t --filename --rate 120 /newton/prep "
  373. "tests/data/prep-20120323T1002")
  374. # overlap
  375. os.environ['TZ'] = "UTC"
  376. self.fail("insert --timestamp -f --rate 120 /newton/prep "
  377. "tests/data/prep-20120323T1004")
  378. self.contain("overlap")
  379. # Just to help test more situations -- stop and restart
  380. # the server now. This tests nilmdb's interval caching,
  381. # at the very least.
  382. server_stop()
  383. server_start()
  384. # still an overlap if we specify a different start
  385. os.environ['TZ'] = "America/New_York"
  386. self.fail("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep"
  387. " tests/data/prep-20120323T1004")
  388. self.contain("overlap")
  389. # wrong format
  390. os.environ['TZ'] = "UTC"
  391. self.fail("insert -t -r 120 -f /newton/raw "
  392. "tests/data/prep-20120323T1004")
  393. self.contain("error parsing input data")
  394. self.contain("can't parse value")
  395. # too few rows per line
  396. self.ok("create /insert/test float32_20")
  397. self.fail("insert -t -r 120 -f /insert/test "
  398. "tests/data/prep-20120323T1004")
  399. self.contain("error parsing input data")
  400. self.contain("wrong number of values")
  401. self.ok("destroy /insert/test")
  402. # empty data does nothing
  403. self.ok("insert -t -r 120 --start '03/23/2012 06:05:00' /newton/prep "
  404. "/dev/null")
  405. # bad start time
  406. self.fail("insert -t -r 120 --start 'whatever' /newton/prep /dev/null")
  407. # Test negative times
  408. self.ok("insert --start @-10000000000 --end @1000000001 /newton/prep"
  409. " tests/data/timestamped")
  410. self.ok("extract -c /newton/prep --start min --end @1000000001")
  411. self.match("8\n")
  412. self.ok("remove /newton/prep --start min --end @1000000001")
  413. def test_07_detail_extended(self):
  414. # Just count the number of lines, it's probably fine
  415. self.ok("list --detail")
  416. lines_(self.captured, 8)
  417. self.ok("list --detail *prep")
  418. lines_(self.captured, 4)
  419. self.ok("list --detail *prep --start='23 Mar 2012 10:02'")
  420. lines_(self.captured, 3)
  421. self.ok("list --detail *prep --start='23 Mar 2012 10:05'")
  422. lines_(self.captured, 2)
  423. self.ok("list --detail *prep --start='23 Mar 2012 10:05:15'")
  424. lines_(self.captured, 2)
  425. self.contain("10:05:15.000")
  426. self.ok("list --detail *prep --start='23 Mar 2012 10:05:15.50'")
  427. lines_(self.captured, 2)
  428. self.contain("10:05:15.500")
  429. self.ok("list --detail *prep --start='23 Mar 2012 19:05:15.50'")
  430. lines_(self.captured, 2)
  431. self.contain("no intervals")
  432. self.ok("list --detail *prep --start='23 Mar 2012 10:05:15.50'"
  433. + " --end='23 Mar 2012 10:05:15.51'")
  434. lines_(self.captured, 2)
  435. self.contain("10:05:15.500")
  436. self.ok("list --detail")
  437. lines_(self.captured, 8)
  438. # Verify the "raw timestamp" output
  439. self.ok("list --detail *prep --timestamp-raw "
  440. "--start='23 Mar 2012 10:05:15.50'")
  441. lines_(self.captured, 2)
  442. self.contain("[ 1332497115500000 -> 1332497160000000 ]")
  443. # bad time
  444. self.fail("list --detail *prep -T --start='9332497115.612'")
  445. # good time
  446. self.ok("list --detail *prep -T --start='1332497115.612'")
  447. lines_(self.captured, 2)
  448. self.contain("[ 1332497115612000 -> 1332497160000000 ]")
  449. # Check --ext output
  450. self.ok("list --ext")
  451. lines_(self.captured, 9)
  452. self.ok("list -E -T")
  453. c = self.contain
  454. c("\n interval extents: 1332496800000000 -> 1332497160000000\n")
  455. c("\n total data: 43200 rows, 359.983336 seconds\n")
  456. c("\n interval extents: (no data)\n")
  457. c("\n total data: 0 rows, 0.000000 seconds\n")
  458. # Misc
  459. self.fail("list --ext --start='23 Mar 2012 10:05:15.50'")
  460. self.contain("--start and --end only make sense with --detail")
  461. def test_08_extract(self):
  462. # nonexistent stream
  463. self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")
  464. self.contain("error getting stream info")
  465. # reversed range
  466. self.fail("extract -a /newton/prep --start 2020-01-01 --end 2000-01-01")
  467. self.contain("start is after end")
  468. # empty ranges return error 2
  469. self.fail("extract -a /newton/prep " +
  470. "--start '23 Mar 2012 20:00:30' " +
  471. "--end '23 Mar 2012 20:00:31'",
  472. exitcode = 2, require_error = False)
  473. self.contain("no data")
  474. self.fail("extract -a /newton/prep " +
  475. "--start '23 Mar 2012 20:00:30.000001' " +
  476. "--end '23 Mar 2012 20:00:30.000002'",
  477. exitcode = 2, require_error = False)
  478. self.contain("no data")
  479. self.fail("extract -a /newton/prep " +
  480. "--start '23 Mar 2022 10:00:30' " +
  481. "--end '23 Mar 2022 10:00:31'",
  482. exitcode = 2, require_error = False)
  483. self.contain("no data")
  484. # but are ok if we're just counting results
  485. self.ok("extract --count /newton/prep " +
  486. "--start '23 Mar 2012 20:00:30' " +
  487. "--end '23 Mar 2012 20:00:31'")
  488. self.match("0\n")
  489. self.ok("extract -c /newton/prep " +
  490. "--start '23 Mar 2012 20:00:30.000001' " +
  491. "--end '23 Mar 2012 20:00:30.000002'")
  492. self.match("0\n")
  493. # Check various dumps against stored copies of how they should appear
  494. def test(file, start, end, extra=""):
  495. self.ok("extract " + extra + " /newton/prep " +
  496. "--start '23 Mar 2012 " + start + "' " +
  497. "--end '23 Mar 2012 " + end + "'")
  498. self.matchfile("tests/data/extract-" + str(file))
  499. self.ok("extract --count " + extra + " /newton/prep " +
  500. "--start '23 Mar 2012 " + start + "' " +
  501. "--end '23 Mar 2012 " + end + "'")
  502. self.matchfilecount("tests/data/extract-" + str(file))
  503. test(1, "10:00:30", "10:00:31", extra="-a")
  504. test(1, "10:00:30.000000", "10:00:31", extra="-a")
  505. test(2, "10:00:30.000001", "10:00:31")
  506. test(2, "10:00:30.008333", "10:00:31")
  507. test(3, "10:00:30.008333", "10:00:30.008334")
  508. test(3, "10:00:30.008333", "10:00:30.016667")
  509. test(4, "10:00:30.008333", "10:00:30.025")
  510. test(5, "10:00:30", "10:00:31", extra="--annotate --bare")
  511. test(6, "10:00:30", "10:00:31", extra="-b")
  512. test(7, "10:00:30", "10:00:30.999", extra="-a -T")
  513. test(7, "10:00:30", "10:00:30.999", extra="-a --timestamp-raw")
  514. test(8, "10:01:59.9", "10:02:00.1", extra="--markup")
  515. test(8, "10:01:59.9", "10:02:00.1", extra="-m")
  516. # all data put in by tests
  517. self.ok("extract -a /newton/prep --start min --end max")
  518. lines_(self.captured, 43204)
  519. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  520. self.match("43200\n")
  521. # test binary mode
  522. self.fail("extract -c -B /newton/prep -s min -e max")
  523. self.contain("binary cannot be combined")
  524. self.fail("extract -m -B /newton/prep -s min -e max")
  525. self.contain("binary cannot be combined")
  526. self.ok("extract -B /newton/prep -s min -e max")
  527. eq_(len(self.captured), 43200 * (8 + 8*4))
  528. # markup for 3 intervals, plus extra markup lines whenever we had
  529. # a "restart" from the nilmdb.stream_extract function
  530. self.ok("extract -m /newton/prep --start 2000-01-01 --end 2020-01-01")
  531. lines_(self.captured, 43210)
  532. def test_09_truncated(self):
  533. # Test truncated responses by overriding the nilmdb max_results
  534. server_stop()
  535. server_start(max_results = 2)
  536. self.ok("list --detail")
  537. lines_(self.captured, 8)
  538. server_stop()
  539. server_start()
  540. def test_10_remove(self):
  541. # Removing data
  542. # Try nonexistent stream
  543. self.fail("remove /no/such/foo --start 2000-01-01 --end 2020-01-01")
  544. self.contain("no stream matched path")
  545. # empty or backward ranges return errors
  546. self.fail("remove /newton/prep --start 2020-01-01 --end 2000-01-01")
  547. self.contain("start must precede end")
  548. self.fail("remove /newton/prep " +
  549. "--start '23 Mar 2012 10:00:30' " +
  550. "--end '23 Mar 2012 10:00:30'")
  551. self.contain("start must precede end")
  552. self.fail("remove /newton/prep " +
  553. "--start '23 Mar 2012 10:00:30.000001' " +
  554. "--end '23 Mar 2012 10:00:30.000001'")
  555. self.contain("start must precede end")
  556. self.fail("remove /newton/prep " +
  557. "--start '23 Mar 2022 10:00:30' " +
  558. "--end '23 Mar 2022 10:00:30'")
  559. self.contain("start must precede end")
  560. # Verbose
  561. self.ok("remove -c /newton/prep " +
  562. "--start '23 Mar 2022 20:00:30' " +
  563. "--end '23 Mar 2022 20:00:31'")
  564. self.match("0\n")
  565. self.ok("remove --count /newton/prep " +
  566. "--start '23 Mar 2022 20:00:30' " +
  567. "--end '23 Mar 2022 20:00:31'")
  568. self.match("0\n")
  569. self.ok("remove -c /newton/prep /newton/pre* " +
  570. "--start '23 Mar 2022 20:00:30' " +
  571. "--end '23 Mar 2022 20:00:31'")
  572. self.match("Removing from /newton/prep\n0\n" +
  573. "Removing from /newton/prep\n0\n")
  574. # Make sure we have the data we expect
  575. self.ok("list -l --detail /newton/prep")
  576. self.match("/newton/prep float32_8\n" +
  577. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  578. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  579. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  580. " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n"
  581. " [ Fri, 23 Mar 2012 10:04:00.000000 +0000"
  582. " -> Fri, 23 Mar 2012 10:06:00.000000 +0000 ]\n")
  583. # Remove various chunks of prep data and make sure
  584. # they're gone.
  585. self.ok("remove -c /newton/prep " +
  586. "--start '23 Mar 2012 10:00:30' " +
  587. "--end '23 Mar 2012 10:00:40'")
  588. self.match("1200\n")
  589. self.ok("remove -c /newton/prep " +
  590. "--start '23 Mar 2012 10:00:10' " +
  591. "--end '23 Mar 2012 10:00:20'")
  592. self.match("1200\n")
  593. self.ok("remove -c /newton/prep " +
  594. "--start '23 Mar 2012 10:00:05' " +
  595. "--end '23 Mar 2012 10:00:25'")
  596. self.match("1200\n")
  597. self.ok("remove -c /newton/prep " +
  598. "--start '23 Mar 2012 10:03:50' " +
  599. "--end '23 Mar 2012 10:06:50'")
  600. self.match("15600\n")
  601. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  602. self.match("24000\n")
  603. # See the missing chunks in list output
  604. self.ok("list --layout --detail /newton/prep")
  605. self.match("/newton/prep float32_8\n" +
  606. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  607. " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
  608. " [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
  609. " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
  610. " [ Fri, 23 Mar 2012 10:00:40.000000 +0000"
  611. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  612. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  613. " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
  614. # Remove all data, verify it's missing
  615. self.ok("remove /newton/prep --start 2000-01-01 --end 2020-01-01")
  616. self.match("") # no count requested this time
  617. self.ok("list -l --detail /newton/prep")
  618. self.match("/newton/prep float32_8\n" +
  619. " (no intervals)\n")
  620. # Reinsert some data, to verify that no overlaps with deleted
  621. # data are reported
  622. for minute in ["0", "2"]:
  623. self.ok("insert --timestamp -f --rate 120 /newton/prep"
  624. " tests/data/prep-20120323T100" + minute)
  625. def test_11_destroy(self):
  626. # Delete records
  627. self.ok("destroy --help")
  628. self.fail("destroy")
  629. self.contain("too few arguments")
  630. self.fail("destroy /no/such/stream")
  631. self.contain("no stream matched path")
  632. self.fail("destroy -R /no/such/stream")
  633. self.contain("no stream matched path")
  634. self.fail("destroy asdfasdf")
  635. self.contain("no stream matched path")
  636. # From previous tests, we have:
  637. self.ok("list -l")
  638. self.match("/newton/prep float32_8\n"
  639. "/newton/raw uint16_6\n"
  640. "/newton/zzz/rawnotch uint16_9\n")
  641. # Notice how they're not empty
  642. self.ok("list --detail")
  643. lines_(self.captured, 7)
  644. # Fail to destroy because intervals still present
  645. self.fail("destroy /newton/prep")
  646. self.contain("all intervals must be removed")
  647. self.ok("list --detail")
  648. lines_(self.captured, 7)
  649. # Destroy for real
  650. self.ok("destroy -R /n*/prep")
  651. self.ok("list -l")
  652. self.match("/newton/raw uint16_6\n"
  653. "/newton/zzz/rawnotch uint16_9\n")
  654. self.ok("destroy /newton/zzz/rawnotch")
  655. self.ok("list -l")
  656. self.match("/newton/raw uint16_6\n")
  657. self.ok("destroy /newton/raw")
  658. self.ok("create /newton/raw uint16_6")
  659. # Specify --remove with no data
  660. self.ok("destroy --remove /newton/raw")
  661. self.ok("list")
  662. self.match("")
  663. # Re-create a previously deleted location, and some new ones
  664. rebuild = [ "/newton/prep", "/newton/zzz",
  665. "/newton/raw", "/newton/asdf/qwer" ]
  666. for path in rebuild:
  667. # Create the path
  668. self.ok("create " + path + " float32_8")
  669. self.ok("list")
  670. self.contain(path)
  671. # Make sure it was created empty
  672. self.ok("list --detail " + path)
  673. self.contain("(no intervals)")
  674. def test_12_unicode(self):
  675. # Unicode paths.
  676. self.ok("destroy /newton/asdf/qwer")
  677. self.ok("destroy /newton/prep /newton/raw")
  678. self.ok("destroy /newton/zzz")
  679. self.ok("create /düsseldorf/raw uint16_6")
  680. self.ok("list -l --detail")
  681. self.contain("/düsseldorf/raw uint16_6")
  682. self.contain("(no intervals)")
  683. # Unicode metadata
  684. self.ok("metadata /düsseldorf/raw --set α=beta 'γ=δ'")
  685. self.ok("metadata /düsseldorf/raw --update 'α=β ε τ α'")
  686. self.ok("metadata /düsseldorf/raw")
  687. self.match("α=β ε τ α\nγ=δ\n")
  688. self.ok("destroy /düsseldorf/raw")
  689. def test_13_files(self):
  690. # Test BulkData's ability to split into multiple files,
  691. # by forcing the file size to be really small.
  692. # Also increase the initial nrows, so that start/end positions
  693. # in the database are very large (> 32 bit)
  694. server_stop()
  695. server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
  696. "files_per_dir" : 3,
  697. "initial_nrows" : 2**40 })
  698. # Fill data
  699. self.ok("create /newton/prep float32_8")
  700. os.environ['TZ'] = "UTC"
  701. with open("tests/data/prep-20120323T1004-timestamped") as input:
  702. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  703. input)
  704. # Extract it
  705. self.ok("extract /newton/prep --start '2000-01-01' " +
  706. "--end '2012-03-23 10:04:01'")
  707. lines_(self.captured, 120)
  708. self.ok("extract /newton/prep --start '2000-01-01' " +
  709. "--end '2022-03-23 10:04:01'")
  710. lines_(self.captured, 14400)
  711. # Make sure there were lots of files generated in the database
  712. # dir
  713. nfiles = 0
  714. for (dirpath, dirnames, filenames) in os.walk(testdb):
  715. nfiles += len(filenames)
  716. assert(nfiles > 500)
  717. # Make sure we can restart the server with a different file
  718. # size and have it still work
  719. server_stop()
  720. server_start()
  721. self.ok("extract /newton/prep --start '2000-01-01' " +
  722. "--end '2022-03-23 10:04:01'")
  723. lines_(self.captured, 14400)
  724. # Now recreate the data one more time and make sure there are
  725. # fewer files.
  726. self.ok("destroy --remove /newton/prep")
  727. self.fail("destroy /newton/prep") # already destroyed
  728. self.ok("create /newton/prep float32_8")
  729. os.environ['TZ'] = "UTC"
  730. with open("tests/data/prep-20120323T1004-timestamped") as input:
  731. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  732. input)
  733. nfiles = 0
  734. for (dirpath, dirnames, filenames) in os.walk(testdb):
  735. nfiles += len(filenames)
  736. lt_(nfiles, 50)
  737. self.ok("destroy -R /newton/prep") # destroy again
  738. def test_14_remove_files(self):
  739. # Limit max_removals, to cover more functionality.
  740. server_stop()
  741. server_start(max_removals = 4321,
  742. bulkdata_args = { "file_size" : 920, # 23 rows per file
  743. "files_per_dir" : 3,
  744. "initial_nrows" : 2**40 })
  745. self.do_remove_files()
  746. self.ok("destroy -R /newton/prep") # destroy again
  747. def test_14b_remove_files_maxint(self):
  748. # Limit max_int_removals, to cover more functionality.
  749. server_stop()
  750. server_start(max_int_removals = 1,
  751. bulkdata_args = { "file_size" : 920, # 23 rows per file
  752. "files_per_dir" : 3,
  753. "initial_nrows" : 2**40 })
  754. self.do_remove_files()
  755. def do_remove_files(self):
  756. # Test BulkData's ability to remove when data is split into
  757. # multiple files. Should be a fairly comprehensive test of
  758. # remove functionality.
  759. # Insert data. Just for fun, insert out of order
  760. self.ok("create /newton/prep float32_8")
  761. os.environ['TZ'] = "UTC"
  762. self.ok("insert -t --filename --rate 120 /newton/prep "
  763. "tests/data/prep-20120323T1002")
  764. self.ok("insert -t --filename --rate 120 /newton/prep "
  765. "tests/data/prep-20120323T1000")
  766. # Should take up about 2.8 MB here (including directory entries)
  767. du_before = nilmdb.utils.diskusage.du(testdb)
  768. # Make sure we have the data we expect
  769. self.ok("list -l --detail")
  770. self.match("/newton/prep float32_8\n" +
  771. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  772. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  773. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  774. " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n")
  775. # Remove various chunks of prep data and make sure
  776. # they're gone.
  777. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  778. self.match("28800\n")
  779. self.ok("remove -c /newton/prep " +
  780. "--start '23 Mar 2012 10:00:30' " +
  781. "--end '23 Mar 2012 10:03:30'")
  782. self.match("21600\n")
  783. self.ok("remove -c /newton/prep " +
  784. "--start '23 Mar 2012 10:00:10' " +
  785. "--end '23 Mar 2012 10:00:20'")
  786. self.match("1200\n")
  787. self.ok("remove -c /newton/prep " +
  788. "--start '23 Mar 2012 10:00:05' " +
  789. "--end '23 Mar 2012 10:00:25'")
  790. self.match("1200\n")
  791. self.ok("remove -c /newton/prep " +
  792. "--start '23 Mar 2012 10:03:50' " +
  793. "--end '23 Mar 2012 10:06:50'")
  794. self.match("1200\n")
  795. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  796. self.match("3600\n")
  797. # See the missing chunks in list output
  798. self.ok("list -l --detail")
  799. self.match("/newton/prep float32_8\n" +
  800. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  801. " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
  802. " [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
  803. " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
  804. " [ Fri, 23 Mar 2012 10:03:30.000000 +0000"
  805. " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
  806. # We have 1/8 of the data that we had before, so the file size
  807. # should have dropped below 1/4 of what it used to be
  808. du_after = nilmdb.utils.diskusage.du(testdb)
  809. lt_(du_after, (du_before / 4))
  810. # Remove anything that came from the 10:02 data file
  811. self.ok("remove /newton/prep " +
  812. "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
  813. # Re-insert 19 lines from that file, then remove them again.
  814. # With the specific file_size above, this will cause the last
  815. # file in the bulk data storage to be exactly file_size large,
  816. # so removing the data should also remove that last file.
  817. self.ok("insert --timestamp -f --rate 120 /newton/prep " +
  818. "tests/data/prep-20120323T1002-first19lines")
  819. self.ok("remove /newton/prep " +
  820. "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
  821. # Shut down and restart server, to force nrows to get refreshed.
  822. server_stop()
  823. server_start()
  824. # Re-add the full 10:02 data file. This tests adding new data once
  825. # we removed data near the end.
  826. self.ok("insert -t -f -r 120 /newton/prep "
  827. "tests/data/prep-20120323T1002")
  828. # See if we can extract it all
  829. self.ok("extract /newton/prep --start 2000-01-01 --end 2020-01-01")
  830. lines_(self.captured, 15600)
  831. def test_15_intervals_diff(self):
  832. # Test "intervals" and "intervals --diff" command.
  833. os.environ['TZ'] = "UTC"
  834. self.ok("create /diff/1 uint8_1")
  835. self.match("")
  836. self.ok("intervals /diff/1")
  837. self.match("")
  838. self.ok("intervals /diff/1 --diff /diff/1")
  839. self.match("")
  840. self.ok("intervals --diff /diff/1 /diff/1")
  841. self.match("")
  842. self.fail("intervals /diff/2")
  843. self.fail("intervals /diff/1 -d /diff/2")
  844. self.ok("create /diff/2 uint8_1")
  845. self.ok("intervals -T /diff/1 -d /diff/2")
  846. self.match("")
  847. self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/1 /dev/null")
  848. self.ok("intervals /diff/1")
  849. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  850. "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
  851. self.ok("intervals /diff/1 -d /diff/2")
  852. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  853. "> Mon, 01 Jan 2001 00:00:00.000000 +0000 ]\n")
  854. self.ok("insert -s 01-01-2000 -e 01-01-2001 /diff/2 /dev/null")
  855. self.ok("intervals /diff/1 -d /diff/2")
  856. self.match("")
  857. self.ok("insert -s 01-01-2001 -e 01-01-2002 /diff/1 /dev/null")
  858. self.ok("insert -s 01-01-2002 -e 01-01-2003 /diff/2 /dev/null")
  859. self.ok("intervals /diff/1 -d /diff/2")
  860. self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
  861. "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n")
  862. self.ok("insert -s 01-01-2004 -e 01-01-2005 /diff/1 /dev/null")
  863. self.ok("intervals /diff/1 -d /diff/2")
  864. self.match("[ Mon, 01 Jan 2001 00:00:00.000000 +0000 -"
  865. "> Tue, 01 Jan 2002 00:00:00.000000 +0000 ]\n"
  866. "[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
  867. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  868. self.fail("intervals -s 01-01-2003 -e 01-01-2000 /diff/1 -d /diff/2")
  869. self.ok("intervals -s 01-01-2003 -e 01-01-2008 /diff/1 -d /diff/2")
  870. self.match("[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
  871. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  872. # optimize
  873. self.ok("insert -s 01-01-2002 -e 01-01-2004 /diff/1 /dev/null")
  874. self.ok("intervals /diff/1")
  875. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  876. "> Thu, 01 Jan 2004 00:00:00.000000 +0000 ]\n"
  877. "[ Thu, 01 Jan 2004 00:00:00.000000 +0000 -"
  878. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  879. self.ok("intervals /diff/1 --optimize")
  880. self.ok("intervals /diff/1 -o")
  881. self.match("[ Sat, 01 Jan 2000 00:00:00.000000 +0000 -"
  882. "> Sat, 01 Jan 2005 00:00:00.000000 +0000 ]\n")
  883. self.ok("destroy -R /diff/1")
  884. self.ok("destroy -R /diff/2")
  885. def test_16_rename(self):
  886. # Test renaming. Force file size smaller so we get more files
  887. server_stop()
  888. recursive_unlink(testdb)
  889. server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
  890. "files_per_dir" : 3 })
  891. # Fill data
  892. self.ok("create /newton/prep float32_8")
  893. os.environ['TZ'] = "UTC"
  894. with open("tests/data/prep-20120323T1004-timestamped") as input:
  895. self.ok("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
  896. input)
  897. # Extract it
  898. self.ok("extract /newton/prep --start '2000-01-01' " +
  899. "--end '2012-03-23 10:04:01'")
  900. extract_before = self.captured
  901. def check_path(*components):
  902. # Verify the paths look right on disk
  903. seek = os.path.join(testdb, "data", *components)
  904. for (dirpath, dirnames, filenames) in os.walk(testdb):
  905. if "_format" in filenames:
  906. if dirpath == seek:
  907. break
  908. raise AssertionError("data also found at " + dirpath)
  909. else:
  910. raise AssertionError("data not found at " + seek)
  911. # Verify "list" output
  912. self.ok("list -l")
  913. self.match("/" + "/".join(components) + " float32_8\n")
  914. # Lots of renames
  915. check_path("newton", "prep")
  916. self.fail("rename /newton/prep /newton/prep")
  917. self.contain("old and new paths are the same")
  918. check_path("newton", "prep")
  919. self.fail("rename /newton/prep /newton")
  920. self.contain("path must contain at least one folder")
  921. self.fail("rename /newton/prep /newton/prep/")
  922. self.contain("invalid path")
  923. self.ok("rename /newton/prep /newton/foo/1")
  924. check_path("newton", "foo", "1")
  925. self.ok("rename /newton/foo/1 /newton/foo")
  926. check_path("newton", "foo")
  927. self.ok("rename /newton/foo /totally/different/thing")
  928. check_path("totally", "different", "thing")
  929. self.ok("rename /totally/different/thing /totally/something")
  930. check_path("totally", "something")
  931. self.ok("rename /totally/something /totally/something/cool")
  932. check_path("totally", "something", "cool")
  933. self.ok("rename /totally/something/cool /foo/bar")
  934. check_path("foo", "bar")
  935. self.ok("create /xxx/yyy/zzz float32_8")
  936. self.fail("rename /foo/bar /xxx/yyy")
  937. self.contain("subdirs of this path already exist")
  938. self.fail("rename /foo/bar /xxx/yyy/zzz")
  939. self.contain("stream already exists at this path")
  940. self.fail("rename /foo/bar /xxx/yyy/zzz/www")
  941. self.contain("path is subdir of existing node")
  942. self.ok("rename /foo/bar /xxx/yyy/mmm")
  943. self.ok("destroy -R /xxx/yyy/zzz")
  944. check_path("xxx", "yyy", "mmm")
  945. # Extract it at the final path
  946. self.ok("extract /xxx/yyy/mmm --start '2000-01-01' " +
  947. "--end '2012-03-23 10:04:01'")
  948. eq_(self.captured, extract_before)
  949. self.ok("destroy -R /xxx/yyy/mmm")
  950. # Make sure temporary rename dirs weren't left around
  951. for (dirpath, dirnames, filenames) in os.walk(testdb):
  952. if "rename-" in dirpath:
  953. raise AssertionError("temporary directories not cleaned up")
  954. if "totally" in dirpath or "newton" in dirpath:
  955. raise AssertionError("old directories not cleaned up")
  956. server_stop()
  957. server_start()