You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

870 lines
32 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb
  3. from nilmdb.utils.printf import *
  4. import nilmdb.cmdline
  5. from nilmdb.utils import datetime_tz
  6. import unittest
  7. from nose.tools import *
  8. from nose.tools import assert_raises
  9. import itertools
  10. import os
  11. import re
  12. import sys
  13. import StringIO
  14. import shlex
  15. from testutil.helpers import *
  16. testdb = "tests/cmdline-testdb"
  17. def server_start(max_results = None, bulkdata_args = {}):
  18. global test_server, test_db
  19. # Start web app on a custom port
  20. test_db = nilmdb.utils.serializer_proxy(nilmdb.NilmDB)(
  21. testdb, sync = False,
  22. max_results = max_results,
  23. bulkdata_args = bulkdata_args)
  24. test_server = nilmdb.Server(test_db, host = "127.0.0.1",
  25. port = 32180, stoppable = False,
  26. fast_shutdown = True,
  27. force_traceback = False)
  28. test_server.start(blocking = False)
  29. def server_stop():
  30. global test_server, test_db
  31. # Close web app
  32. test_server.stop()
  33. test_db.close()
  34. def setup_module():
  35. global test_server, test_db
  36. # Clear out DB
  37. recursive_unlink(testdb)
  38. server_start()
  39. def teardown_module():
  40. server_stop()
  41. # Add an encoding property to StringIO so Python will convert Unicode
  42. # properly when writing or reading.
  43. class UTF8StringIO(StringIO.StringIO):
  44. encoding = 'utf-8'
  45. class TestCmdline(object):
  46. def run(self, arg_string, infile=None, outfile=None):
  47. """Run a cmdline client with the specified argument string,
  48. passing the given input. Returns a tuple with the output and
  49. exit code"""
  50. # printf("TZ=UTC ./nilmtool.py %s\n", arg_string)
  51. os.environ['NILMDB_URL'] = "http://localhost:32180/"
  52. class stdio_wrapper:
  53. def __init__(self, stdin, stdout, stderr):
  54. self.io = (stdin, stdout, stderr)
  55. def __enter__(self):
  56. self.saved = ( sys.stdin, sys.stdout, sys.stderr )
  57. ( sys.stdin, sys.stdout, sys.stderr ) = self.io
  58. def __exit__(self, type, value, traceback):
  59. ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
  60. # Empty input if none provided
  61. if infile is None:
  62. infile = UTF8StringIO("")
  63. # Capture stderr
  64. errfile = UTF8StringIO()
  65. if outfile is None:
  66. # If no output file, capture stdout with stderr
  67. outfile = errfile
  68. with stdio_wrapper(infile, outfile, errfile) as s:
  69. try:
  70. # shlex doesn't support Unicode very well. Encode the
  71. # string as UTF-8 explicitly before splitting.
  72. args = shlex.split(arg_string.encode('utf-8'))
  73. nilmdb.cmdline.Cmdline(args).run()
  74. sys.exit(0)
  75. except SystemExit as e:
  76. exitcode = e.code
  77. captured = outfile.getvalue()
  78. self.captured = captured
  79. self.exitcode = exitcode
  80. def ok(self, arg_string, infile = None):
  81. self.run(arg_string, infile)
  82. if self.exitcode != 0:
  83. self.dump()
  84. eq_(self.exitcode, 0)
  85. def fail(self, arg_string, infile = None,
  86. exitcode = None, require_error = True):
  87. self.run(arg_string, infile)
  88. if exitcode is not None and self.exitcode != exitcode:
  89. # Wrong exit code
  90. self.dump()
  91. eq_(self.exitcode, exitcode)
  92. if self.exitcode == 0:
  93. # Success, when we wanted failure
  94. self.dump()
  95. ne_(self.exitcode, 0)
  96. # Make sure the output contains the word "error" at the
  97. # beginning of a line, but only if an exitcode wasn't
  98. # specified.
  99. if require_error and not re.search("^error",
  100. self.captured, re.MULTILINE):
  101. raise AssertionError("command failed, but output doesn't "
  102. "contain the string 'error'")
  103. def contain(self, checkstring):
  104. in_(checkstring, self.captured)
  105. def match(self, checkstring):
  106. eq_(checkstring, self.captured)
  107. def matchfile(self, file):
  108. # Captured data should match file contents exactly
  109. with open(file) as f:
  110. contents = f.read()
  111. if contents != self.captured:
  112. print contents[1:1000] + "\n"
  113. print self.captured[1:1000] + "\n"
  114. raise AssertionError("captured data doesn't match " + file)
  115. def matchfilecount(self, file):
  116. # Last line of captured data should match the number of
  117. # non-commented lines in file
  118. count = 0
  119. with open(file) as f:
  120. for line in f:
  121. if line[0] != '#':
  122. count += 1
  123. eq_(self.captured.splitlines()[-1], sprintf("%d", count))
  124. def dump(self):
  125. printf("-----dump start-----\n%s-----dump end-----\n", self.captured)
  126. def test_01_basic(self):
  127. # help
  128. self.ok("--help")
  129. self.contain("usage:")
  130. # fail for no args
  131. self.fail("")
  132. # fail for no such option
  133. self.fail("--nosuchoption")
  134. # fail for bad command
  135. self.fail("badcommand")
  136. # try some URL constructions
  137. self.fail("--url http://nosuchurl/ info")
  138. self.contain("error connecting to server")
  139. self.fail("--url nosuchurl info")
  140. self.contain("error connecting to server")
  141. self.fail("-u nosuchurl/foo info")
  142. self.contain("error connecting to server")
  143. self.fail("-u localhost:1 info")
  144. self.contain("error connecting to server")
  145. self.ok("-u localhost:32180 info")
  146. self.ok("info")
  147. # Duplicated arguments should fail, but this isn't implemented
  148. # due to it being kind of a pain with argparse.
  149. if 0:
  150. self.fail("-u url1 -u url2 info")
  151. self.contain("duplicated argument")
  152. self.fail("list --detail --detail")
  153. self.contain("duplicated argument")
  154. self.fail("list --detail --path path1 --path path2")
  155. self.contain("duplicated argument")
  156. self.fail("extract --start 2000-01-01 --start 2001-01-02")
  157. self.contain("duplicated argument")
  158. # Verify that "help command" and "command --help" are identical
  159. # for all commands.
  160. self.fail("")
  161. m = re.search(r"{(.*)}", self.captured)
  162. for command in [""] + m.group(1).split(','):
  163. self.ok(command + " --help")
  164. cap1 = self.captured
  165. self.ok("help " + command)
  166. cap2 = self.captured
  167. self.ok("help " + command + " asdf --url --zxcv -")
  168. cap3 = self.captured
  169. eq_(cap1, cap2)
  170. eq_(cap2, cap3)
  171. def test_02_parsetime(self):
  172. os.environ['TZ'] = "America/New_York"
  173. test = datetime_tz.datetime_tz.now()
  174. parse_time = nilmdb.utils.time.parse_time
  175. eq_(parse_time(str(test)), test)
  176. test = datetime_tz.datetime_tz.smartparse("20120405 1400-0400")
  177. eq_(parse_time("hi there 20120405 1400-0400 testing! 123"), test)
  178. eq_(parse_time("20120405 1800 UTC"), test)
  179. eq_(parse_time("20120405 1400-0400 UTC"), test)
  180. for badtime in [ "20120405 1400-9999", "hello", "-", "", "4:00" ]:
  181. with assert_raises(ValueError):
  182. x = parse_time(badtime)
  183. x = parse_time("now")
  184. eq_(parse_time("snapshot-20120405-140000.raw.gz"), test)
  185. eq_(parse_time("prep-20120405T1400"), test)
  186. def test_03_info(self):
  187. self.ok("info")
  188. self.contain("Server URL: http://localhost:32180/")
  189. self.contain("Client version: " + nilmdb.__version__)
  190. self.contain("Server version: " + test_server.version)
  191. self.contain("Server database path")
  192. self.contain("Server database size")
  193. self.contain("Server database free space")
  194. def test_04_createlist(self):
  195. # Basic stream tests, like those in test_client.
  196. # No streams
  197. self.ok("list")
  198. self.match("")
  199. # Bad paths
  200. self.fail("create foo/bar/baz PrepData")
  201. self.contain("paths must start with /")
  202. self.fail("create /foo PrepData")
  203. self.contain("invalid path")
  204. # Bad layout type
  205. self.fail("create /newton/prep NoSuchLayout")
  206. self.contain("no such layout")
  207. self.fail("create /newton/prep float32_0")
  208. self.contain("no such layout")
  209. self.fail("create /newton/prep float33_1")
  210. self.contain("no such layout")
  211. # Create a few streams
  212. self.ok("create /newton/zzz/rawnotch RawNotchedData")
  213. self.ok("create /newton/prep PrepData")
  214. self.ok("create /newton/raw RawData")
  215. # Should not be able to create a stream with another stream as
  216. # its parent
  217. self.fail("create /newton/prep/blah PrepData")
  218. self.contain("path is subdir of existing node")
  219. # Should not be able to create a stream at a location that
  220. # has other nodes as children
  221. self.fail("create /newton/zzz PrepData")
  222. self.contain("subdirs of this path already exist")
  223. # Verify we got those 3 streams and they're returned in
  224. # alphabetical order.
  225. self.ok("list")
  226. self.match("/newton/prep PrepData\n"
  227. "/newton/raw RawData\n"
  228. "/newton/zzz/rawnotch RawNotchedData\n")
  229. # Match just one type or one path. Also check
  230. # that --path is optional
  231. self.ok("list --path /newton/raw")
  232. self.match("/newton/raw RawData\n")
  233. self.ok("list /newton/raw")
  234. self.match("/newton/raw RawData\n")
  235. self.fail("list -p /newton/raw /newton/raw")
  236. self.contain("too many paths")
  237. self.ok("list --layout RawData")
  238. self.match("/newton/raw RawData\n")
  239. # Wildcard matches
  240. self.ok("list --layout Raw*")
  241. self.match("/newton/raw RawData\n"
  242. "/newton/zzz/rawnotch RawNotchedData\n")
  243. self.ok("list --path *zzz* --layout Raw*")
  244. self.match("/newton/zzz/rawnotch RawNotchedData\n")
  245. self.ok("list *zzz* --layout Raw*")
  246. self.match("/newton/zzz/rawnotch RawNotchedData\n")
  247. self.ok("list --path *zzz* --layout Prep*")
  248. self.match("")
  249. # reversed range
  250. self.fail("list /newton/prep --start 2020-01-01 --end 2000-01-01")
  251. self.contain("start must precede end")
  252. def test_05_metadata(self):
  253. # Set / get metadata
  254. self.fail("metadata")
  255. self.fail("metadata --get")
  256. self.ok("metadata /newton/prep")
  257. self.match("")
  258. self.ok("metadata /newton/raw --get")
  259. self.match("")
  260. self.ok("metadata /newton/prep --set "
  261. "'description=The Data' "
  262. "v_scale=1.234")
  263. self.ok("metadata /newton/raw --update "
  264. "'description=The Data'")
  265. self.ok("metadata /newton/raw --update "
  266. "v_scale=1.234")
  267. # various parsing tests
  268. self.ok("metadata /newton/raw --update foo=")
  269. self.fail("metadata /newton/raw --update =bar")
  270. self.fail("metadata /newton/raw --update foo==bar")
  271. self.fail("metadata /newton/raw --update foo;bar")
  272. # errors
  273. self.fail("metadata /newton/nosuchstream foo=bar")
  274. self.contain("unrecognized arguments")
  275. self.fail("metadata /newton/nosuchstream")
  276. self.contain("No stream at path")
  277. self.fail("metadata /newton/nosuchstream --set foo=bar")
  278. self.contain("No stream at path")
  279. self.ok("metadata /newton/prep")
  280. self.match("description=The Data\nv_scale=1.234\n")
  281. self.ok("metadata /newton/prep --get")
  282. self.match("description=The Data\nv_scale=1.234\n")
  283. self.ok("metadata /newton/prep --get descr")
  284. self.match("descr=\n")
  285. self.ok("metadata /newton/prep --get description")
  286. self.match("description=The Data\n")
  287. self.ok("metadata /newton/prep --get description v_scale")
  288. self.match("description=The Data\nv_scale=1.234\n")
  289. self.ok("metadata /newton/prep --set "
  290. "'description=The Data'")
  291. self.ok("metadata /newton/prep --get")
  292. self.match("description=The Data\n")
  293. self.fail("metadata /newton/nosuchpath")
  294. self.contain("No stream at path /newton/nosuchpath")
  295. def test_06_insert(self):
  296. self.ok("insert --help")
  297. self.fail("insert /foo/bar baz qwer")
  298. self.contain("error getting stream info")
  299. self.fail("insert /newton/prep baz qwer")
  300. self.match("error opening input file baz\n")
  301. self.fail("insert /newton/prep")
  302. self.contain("error extracting time")
  303. self.fail("insert --start 19801205 /newton/prep 1 2 3 4")
  304. self.contain("--start can only be used with one input file")
  305. self.fail("insert /newton/prep "
  306. "tests/data/prep-20120323T1000")
  307. # insert pre-timestamped data, from stdin
  308. os.environ['TZ'] = "UTC"
  309. with open("tests/data/prep-20120323T1004-timestamped") as input:
  310. self.ok("insert --none /newton/prep", input)
  311. # insert pre-timestamped data, with bad times (non-monotonic)
  312. os.environ['TZ'] = "UTC"
  313. with open("tests/data/prep-20120323T1004-badtimes") as input:
  314. self.fail("insert --none /newton/prep", input)
  315. self.contain("error parsing input data")
  316. self.contain("line 7:")
  317. self.contain("timestamp is not monotonically increasing")
  318. # insert data with normal timestamper from filename
  319. os.environ['TZ'] = "UTC"
  320. self.ok("insert --rate 120 /newton/prep "
  321. "tests/data/prep-20120323T1000 "
  322. "tests/data/prep-20120323T1002")
  323. # overlap
  324. os.environ['TZ'] = "UTC"
  325. self.fail("insert --rate 120 /newton/prep "
  326. "tests/data/prep-20120323T1004")
  327. self.contain("overlap")
  328. # Just to help test more situations -- stop and restart
  329. # the server now. This tests nilmdb's interval caching,
  330. # at the very least.
  331. server_stop()
  332. server_start()
  333. # still an overlap if we specify a different start
  334. os.environ['TZ'] = "America/New_York"
  335. self.fail("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep"
  336. " tests/data/prep-20120323T1004")
  337. self.contain("overlap")
  338. # wrong format
  339. os.environ['TZ'] = "UTC"
  340. self.fail("insert --rate 120 /newton/raw "
  341. "tests/data/prep-20120323T1004")
  342. self.contain("error parsing input data")
  343. # empty data does nothing
  344. self.ok("insert --rate 120 --start '03/23/2012 06:05:00' /newton/prep "
  345. "/dev/null")
  346. # bad start time
  347. self.fail("insert --rate 120 --start 'whatever' /newton/prep /dev/null")
  348. def test_07_detail_extent(self):
  349. # Just count the number of lines, it's probably fine
  350. self.ok("list --detail")
  351. lines_(self.captured, 8)
  352. self.ok("list --detail --path *prep")
  353. lines_(self.captured, 4)
  354. self.ok("list --detail --path *prep --start='23 Mar 2012 10:02'")
  355. lines_(self.captured, 3)
  356. self.ok("list --detail --path *prep --start='23 Mar 2012 10:05'")
  357. lines_(self.captured, 2)
  358. self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15'")
  359. lines_(self.captured, 2)
  360. self.contain("10:05:15.000")
  361. self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'")
  362. lines_(self.captured, 2)
  363. self.contain("10:05:15.500")
  364. self.ok("list --detail --path *prep --start='23 Mar 2012 19:05:15.50'")
  365. lines_(self.captured, 2)
  366. self.contain("no intervals")
  367. self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'"
  368. + " --end='23 Mar 2012 10:05:15.51'")
  369. lines_(self.captured, 2)
  370. self.contain("10:05:15.500")
  371. self.ok("list --detail")
  372. lines_(self.captured, 8)
  373. # Verify the "raw timestamp" output
  374. self.ok("list --detail --path *prep --timestamp-raw "
  375. "--start='23 Mar 2012 10:05:15.50'")
  376. lines_(self.captured, 2)
  377. self.contain("[ 1332497115.5 -> 1332497159.991668 ]")
  378. self.ok("list --detail --path *prep -T "
  379. "--start='23 Mar 2012 10:05:15.612'")
  380. lines_(self.captured, 2)
  381. self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
  382. # Check --extent output
  383. self.ok("list --extent")
  384. lines_(self.captured, 6)
  385. self.ok("list -E -T")
  386. self.contain(" extent: 1332496800 -> 1332497159.991668")
  387. self.contain(" extent: (no data)")
  388. # Misc
  389. self.fail("list --extent --start='23 Mar 2012 10:05:15.50'")
  390. self.contain("--start and --end only make sense with --detail")
  391. def test_08_extract(self):
  392. # nonexistent stream
  393. self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")
  394. self.contain("error getting stream info")
  395. # reversed range
  396. self.fail("extract -a /newton/prep --start 2020-01-01 --end 2000-01-01")
  397. self.contain("start is after end")
  398. # empty ranges return error 2
  399. self.fail("extract -a /newton/prep " +
  400. "--start '23 Mar 2012 20:00:30' " +
  401. "--end '23 Mar 2012 20:00:31'",
  402. exitcode = 2, require_error = False)
  403. self.contain("no data")
  404. self.fail("extract -a /newton/prep " +
  405. "--start '23 Mar 2012 20:00:30.000001' " +
  406. "--end '23 Mar 2012 20:00:30.000002'",
  407. exitcode = 2, require_error = False)
  408. self.contain("no data")
  409. self.fail("extract -a /newton/prep " +
  410. "--start '23 Mar 2022 10:00:30' " +
  411. "--end '23 Mar 2022 10:00:31'",
  412. exitcode = 2, require_error = False)
  413. self.contain("no data")
  414. # but are ok if we're just counting results
  415. self.ok("extract --count /newton/prep " +
  416. "--start '23 Mar 2012 20:00:30' " +
  417. "--end '23 Mar 2012 20:00:31'")
  418. self.match("0\n")
  419. self.ok("extract -c /newton/prep " +
  420. "--start '23 Mar 2012 20:00:30.000001' " +
  421. "--end '23 Mar 2012 20:00:30.000002'")
  422. self.match("0\n")
  423. # Check various dumps against stored copies of how they should appear
  424. def test(file, start, end, extra=""):
  425. self.ok("extract " + extra + " /newton/prep " +
  426. "--start '23 Mar 2012 " + start + "' " +
  427. "--end '23 Mar 2012 " + end + "'")
  428. self.matchfile("tests/data/extract-" + str(file))
  429. self.ok("extract --count " + extra + " /newton/prep " +
  430. "--start '23 Mar 2012 " + start + "' " +
  431. "--end '23 Mar 2012 " + end + "'")
  432. self.matchfilecount("tests/data/extract-" + str(file))
  433. test(1, "10:00:30", "10:00:31", extra="-a")
  434. test(1, "10:00:30.000000", "10:00:31", extra="-a")
  435. test(2, "10:00:30.000001", "10:00:31")
  436. test(2, "10:00:30.008333", "10:00:31")
  437. test(3, "10:00:30.008333", "10:00:30.008334")
  438. test(3, "10:00:30.008333", "10:00:30.016667")
  439. test(4, "10:00:30.008333", "10:00:30.025")
  440. test(5, "10:00:30", "10:00:31", extra="--annotate --bare")
  441. test(6, "10:00:30", "10:00:31", extra="-b")
  442. test(7, "10:00:30", "10:00:30.999", extra="-a -T")
  443. test(7, "10:00:30", "10:00:30.999", extra="-a --timestamp-raw")
  444. # all data put in by tests
  445. self.ok("extract -a /newton/prep --start 2000-01-01 --end 2020-01-01")
  446. lines_(self.captured, 43204)
  447. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  448. self.match("43200\n")
  449. def test_09_truncated(self):
  450. # Test truncated responses by overriding the nilmdb max_results
  451. server_stop()
  452. server_start(max_results = 2)
  453. self.ok("list --detail")
  454. lines_(self.captured, 8)
  455. server_stop()
  456. server_start()
  457. def test_10_remove(self):
  458. # Removing data
  459. # Try nonexistent stream
  460. self.fail("remove /no/such/foo --start 2000-01-01 --end 2020-01-01")
  461. self.contain("No stream at path")
  462. # empty or backward ranges return errors
  463. self.fail("remove /newton/prep --start 2020-01-01 --end 2000-01-01")
  464. self.contain("start must precede end")
  465. self.fail("remove /newton/prep " +
  466. "--start '23 Mar 2012 10:00:30' " +
  467. "--end '23 Mar 2012 10:00:30'")
  468. self.contain("start must precede end")
  469. self.fail("remove /newton/prep " +
  470. "--start '23 Mar 2012 10:00:30.000001' " +
  471. "--end '23 Mar 2012 10:00:30.000001'")
  472. self.contain("start must precede end")
  473. self.fail("remove /newton/prep " +
  474. "--start '23 Mar 2022 10:00:30' " +
  475. "--end '23 Mar 2022 10:00:30'")
  476. self.contain("start must precede end")
  477. # Verbose
  478. self.ok("remove -c /newton/prep " +
  479. "--start '23 Mar 2022 20:00:30' " +
  480. "--end '23 Mar 2022 20:00:31'")
  481. self.match("0\n")
  482. self.ok("remove --count /newton/prep " +
  483. "--start '23 Mar 2022 20:00:30' " +
  484. "--end '23 Mar 2022 20:00:31'")
  485. self.match("0\n")
  486. # Make sure we have the data we expect
  487. self.ok("list --detail /newton/prep")
  488. self.match("/newton/prep PrepData\n" +
  489. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  490. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  491. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  492. " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n"
  493. " [ Fri, 23 Mar 2012 10:04:00.000000 +0000"
  494. " -> Fri, 23 Mar 2012 10:05:59.991668 +0000 ]\n")
  495. # Remove various chunks of prep data and make sure
  496. # they're gone.
  497. self.ok("remove -c /newton/prep " +
  498. "--start '23 Mar 2012 10:00:30' " +
  499. "--end '23 Mar 2012 10:00:40'")
  500. self.match("1200\n")
  501. self.ok("remove -c /newton/prep " +
  502. "--start '23 Mar 2012 10:00:10' " +
  503. "--end '23 Mar 2012 10:00:20'")
  504. self.match("1200\n")
  505. self.ok("remove -c /newton/prep " +
  506. "--start '23 Mar 2012 10:00:05' " +
  507. "--end '23 Mar 2012 10:00:25'")
  508. self.match("1200\n")
  509. self.ok("remove -c /newton/prep " +
  510. "--start '23 Mar 2012 10:03:50' " +
  511. "--end '23 Mar 2012 10:06:50'")
  512. self.match("15600\n")
  513. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  514. self.match("24000\n")
  515. # See the missing chunks in list output
  516. self.ok("list --detail /newton/prep")
  517. self.match("/newton/prep PrepData\n" +
  518. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  519. " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
  520. " [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
  521. " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
  522. " [ Fri, 23 Mar 2012 10:00:40.000000 +0000"
  523. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  524. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  525. " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
  526. # Remove all data, verify it's missing
  527. self.ok("remove /newton/prep --start 2000-01-01 --end 2020-01-01")
  528. self.match("") # no count requested this time
  529. self.ok("list --detail /newton/prep")
  530. self.match("/newton/prep PrepData\n" +
  531. " (no intervals)\n")
  532. # Reinsert some data, to verify that no overlaps with deleted
  533. # data are reported
  534. os.environ['TZ'] = "UTC"
  535. self.ok("insert --rate 120 /newton/prep "
  536. "tests/data/prep-20120323T1000 "
  537. "tests/data/prep-20120323T1002")
  538. def test_11_destroy(self):
  539. # Delete records
  540. self.ok("destroy --help")
  541. self.fail("destroy")
  542. self.contain("too few arguments")
  543. self.fail("destroy /no/such/stream")
  544. self.contain("No stream at path")
  545. self.fail("destroy asdfasdf")
  546. self.contain("No stream at path")
  547. # From previous tests, we have:
  548. self.ok("list")
  549. self.match("/newton/prep PrepData\n"
  550. "/newton/raw RawData\n"
  551. "/newton/zzz/rawnotch RawNotchedData\n")
  552. # Notice how they're not empty
  553. self.ok("list --detail")
  554. lines_(self.captured, 7)
  555. # Delete some
  556. self.ok("destroy /newton/prep")
  557. self.ok("list")
  558. self.match("/newton/raw RawData\n"
  559. "/newton/zzz/rawnotch RawNotchedData\n")
  560. self.ok("destroy /newton/zzz/rawnotch")
  561. self.ok("list")
  562. self.match("/newton/raw RawData\n")
  563. self.ok("destroy /newton/raw")
  564. self.ok("create /newton/raw RawData")
  565. self.ok("destroy /newton/raw")
  566. self.ok("list")
  567. self.match("")
  568. # Re-create a previously deleted location, and some new ones
  569. rebuild = [ "/newton/prep", "/newton/zzz",
  570. "/newton/raw", "/newton/asdf/qwer" ]
  571. for path in rebuild:
  572. # Create the path
  573. self.ok("create " + path + " PrepData")
  574. self.ok("list")
  575. self.contain(path)
  576. # Make sure it was created empty
  577. self.ok("list --detail --path " + path)
  578. self.contain("(no intervals)")
  579. def test_12_unicode(self):
  580. # Unicode paths.
  581. self.ok("destroy /newton/asdf/qwer")
  582. self.ok("destroy /newton/prep")
  583. self.ok("destroy /newton/raw")
  584. self.ok("destroy /newton/zzz")
  585. self.ok(u"create /düsseldorf/raw uint16_6")
  586. self.ok("list --detail")
  587. self.contain(u"/düsseldorf/raw uint16_6")
  588. self.contain("(no intervals)")
  589. # Unicode metadata
  590. self.ok(u"metadata /düsseldorf/raw --set α=beta 'γ=δ'")
  591. self.ok(u"metadata /düsseldorf/raw --update 'α=β ε τ α'")
  592. self.ok(u"metadata /düsseldorf/raw")
  593. self.match(u"α=β ε τ α\nγ=δ\n")
  594. self.ok(u"destroy /düsseldorf/raw")
  595. def test_13_files(self):
  596. # Test BulkData's ability to split into multiple files,
  597. # by forcing the file size to be really small.
  598. server_stop()
  599. server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
  600. "files_per_dir" : 3 })
  601. # Fill data
  602. self.ok("create /newton/prep float32_8")
  603. os.environ['TZ'] = "UTC"
  604. with open("tests/data/prep-20120323T1004-timestamped") as input:
  605. self.ok("insert --none /newton/prep", input)
  606. # Extract it
  607. self.ok("extract /newton/prep --start '2000-01-01' " +
  608. "--end '2012-03-23 10:04:01'")
  609. lines_(self.captured, 120)
  610. self.ok("extract /newton/prep --start '2000-01-01' " +
  611. "--end '2022-03-23 10:04:01'")
  612. lines_(self.captured, 14400)
  613. # Make sure there were lots of files generated in the database
  614. # dir
  615. nfiles = 0
  616. for (dirpath, dirnames, filenames) in os.walk(testdb):
  617. nfiles += len(filenames)
  618. assert(nfiles > 500)
  619. # Make sure we can restart the server with a different file
  620. # size and have it still work
  621. server_stop()
  622. server_start()
  623. self.ok("extract /newton/prep --start '2000-01-01' " +
  624. "--end '2022-03-23 10:04:01'")
  625. lines_(self.captured, 14400)
  626. # Now recreate the data one more time and make sure there are
  627. # fewer files.
  628. self.ok("destroy /newton/prep")
  629. self.fail("destroy /newton/prep") # already destroyed
  630. self.ok("create /newton/prep float32_8")
  631. os.environ['TZ'] = "UTC"
  632. with open("tests/data/prep-20120323T1004-timestamped") as input:
  633. self.ok("insert --none /newton/prep", input)
  634. nfiles = 0
  635. for (dirpath, dirnames, filenames) in os.walk(testdb):
  636. nfiles += len(filenames)
  637. lt_(nfiles, 50)
  638. self.ok("destroy /newton/prep") # destroy again
  639. def test_14_remove_files(self):
  640. # Test BulkData's ability to remove when data is split into
  641. # multiple files. Should be a fairly comprehensive test of
  642. # remove functionality.
  643. server_stop()
  644. server_start(bulkdata_args = { "file_size" : 920, # 23 rows per file
  645. "files_per_dir" : 3 })
  646. # Insert data. Just for fun, insert out of order
  647. self.ok("create /newton/prep PrepData")
  648. os.environ['TZ'] = "UTC"
  649. self.ok("insert --rate 120 /newton/prep "
  650. "tests/data/prep-20120323T1002 "
  651. "tests/data/prep-20120323T1000")
  652. # Should take up about 2.8 MB here (including directory entries)
  653. du_before = nilmdb.utils.diskusage.du(testdb)
  654. # Make sure we have the data we expect
  655. self.ok("list --detail")
  656. self.match("/newton/prep PrepData\n" +
  657. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  658. " -> Fri, 23 Mar 2012 10:01:59.991668 +0000 ]\n"
  659. " [ Fri, 23 Mar 2012 10:02:00.000000 +0000"
  660. " -> Fri, 23 Mar 2012 10:03:59.991668 +0000 ]\n")
  661. # Remove various chunks of prep data and make sure
  662. # they're gone.
  663. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  664. self.match("28800\n")
  665. self.ok("remove -c /newton/prep " +
  666. "--start '23 Mar 2012 10:00:30' " +
  667. "--end '23 Mar 2012 10:03:30'")
  668. self.match("21600\n")
  669. self.ok("remove -c /newton/prep " +
  670. "--start '23 Mar 2012 10:00:10' " +
  671. "--end '23 Mar 2012 10:00:20'")
  672. self.match("1200\n")
  673. self.ok("remove -c /newton/prep " +
  674. "--start '23 Mar 2012 10:00:05' " +
  675. "--end '23 Mar 2012 10:00:25'")
  676. self.match("1200\n")
  677. self.ok("remove -c /newton/prep " +
  678. "--start '23 Mar 2012 10:03:50' " +
  679. "--end '23 Mar 2012 10:06:50'")
  680. self.match("1200\n")
  681. self.ok("extract -c /newton/prep --start 2000-01-01 --end 2020-01-01")
  682. self.match("3600\n")
  683. # See the missing chunks in list output
  684. self.ok("list --detail")
  685. self.match("/newton/prep PrepData\n" +
  686. " [ Fri, 23 Mar 2012 10:00:00.000000 +0000"
  687. " -> Fri, 23 Mar 2012 10:00:05.000000 +0000 ]\n"
  688. " [ Fri, 23 Mar 2012 10:00:25.000000 +0000"
  689. " -> Fri, 23 Mar 2012 10:00:30.000000 +0000 ]\n"
  690. " [ Fri, 23 Mar 2012 10:03:30.000000 +0000"
  691. " -> Fri, 23 Mar 2012 10:03:50.000000 +0000 ]\n")
  692. # We have 1/8 of the data that we had before, so the file size
  693. # should have dropped below 1/4 of what it used to be
  694. du_after = nilmdb.utils.diskusage.du(testdb)
  695. lt_(du_after, (du_before / 4))
  696. # Remove anything that came from the 10:02 data file
  697. self.ok("remove /newton/prep " +
  698. "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
  699. # Re-insert 19 lines from that file, then remove them again.
  700. # With the specific file_size above, this will cause the last
  701. # file in the bulk data storage to be exactly file_size large,
  702. # so removing the data should also remove that last file.
  703. self.ok("insert --rate 120 /newton/prep " +
  704. "tests/data/prep-20120323T1002-first19lines")
  705. self.ok("remove /newton/prep " +
  706. "--start '23 Mar 2012 10:02:00' --end '2020-01-01'")
  707. # Shut down and restart server, to force nrows to get refreshed.
  708. server_stop()
  709. server_start()
  710. # Re-add the full 10:02 data file. This tests adding new data once
  711. # we removed data near the end.
  712. self.ok("insert --rate 120 /newton/prep tests/data/prep-20120323T1002")
  713. # See if we can extract it all
  714. self.ok("extract /newton/prep --start 2000-01-01 --end 2020-01-01")
  715. lines_(self.captured, 15600)