You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

464 lines
18 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb
  3. from nilmdb.utils.printf import *
  4. from nilmdb.utils import timestamper
  5. from nilmdb.client import ClientError, ServerError
  6. from nilmdb.utils import datetime_tz
  7. from nose.tools import *
  8. from nose.tools import assert_raises
  9. import itertools
  10. import distutils.version
  11. import os
  12. import sys
  13. import threading
  14. import cStringIO
  15. import simplejson as json
  16. import unittest
  17. import warnings
  18. import resource
  19. import time
  20. from testutil.helpers import *
  21. testdb = "tests/client-testdb"
  22. testurl = "http://localhost:12380/"
  23. def setup_module():
  24. global test_server, test_db
  25. # Clear out DB
  26. recursive_unlink(testdb)
  27. # Start web app on a custom port
  28. test_db = nilmdb.NilmDB(testdb, sync = False)
  29. test_server = nilmdb.Server(test_db, host = "127.0.0.1",
  30. port = 12380, stoppable = False,
  31. fast_shutdown = True,
  32. force_traceback = False)
  33. test_server.start(blocking = False)
  34. def teardown_module():
  35. global test_server, test_db
  36. # Close web app
  37. test_server.stop()
  38. test_db.close()
  39. class TestClient(object):
  40. def test_client_01_basic(self):
  41. # Test a fake host
  42. client = nilmdb.Client(url = "http://localhost:1/")
  43. with assert_raises(nilmdb.client.ServerError):
  44. client.version()
  45. client.close()
  46. # Trigger same error with a PUT request
  47. client = nilmdb.Client(url = "http://localhost:1/")
  48. with assert_raises(nilmdb.client.ServerError):
  49. client.version()
  50. client.close()
  51. # Then a fake URL on a real host
  52. client = nilmdb.Client(url = "http://localhost:12380/fake/")
  53. with assert_raises(nilmdb.client.ClientError):
  54. client.version()
  55. client.close()
  56. # Now a real URL with no http:// prefix
  57. client = nilmdb.Client(url = "localhost:12380")
  58. version = client.version()
  59. client.close()
  60. # Now use the real URL
  61. client = nilmdb.Client(url = testurl)
  62. version = client.version()
  63. eq_(distutils.version.LooseVersion(version),
  64. distutils.version.LooseVersion(test_server.version))
  65. # Bad URLs should give 404, not 500
  66. with assert_raises(ClientError):
  67. client.http.get("/stream/create")
  68. client.close()
  69. def test_client_02_createlist(self):
  70. # Basic stream tests, like those in test_nilmdb:test_stream
  71. client = nilmdb.Client(url = testurl)
  72. # Database starts empty
  73. eq_(client.stream_list(), [])
  74. # Bad path
  75. with assert_raises(ClientError):
  76. client.stream_create("foo/bar/baz", "PrepData")
  77. with assert_raises(ClientError):
  78. client.stream_create("/foo", "PrepData")
  79. # Bad layout type
  80. with assert_raises(ClientError):
  81. client.stream_create("/newton/prep", "NoSuchLayout")
  82. # Create three streams
  83. client.stream_create("/newton/prep", "PrepData")
  84. client.stream_create("/newton/raw", "RawData")
  85. client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
  86. # Verify we got 3 streams
  87. eq_(client.stream_list(), [ ["/newton/prep", "PrepData"],
  88. ["/newton/raw", "RawData"],
  89. ["/newton/zzz/rawnotch", "RawNotchedData"]
  90. ])
  91. # Match just one type or one path
  92. eq_(client.stream_list(layout="RawData"),
  93. [ ["/newton/raw", "RawData"] ])
  94. eq_(client.stream_list(path="/newton/raw"),
  95. [ ["/newton/raw", "RawData"] ])
  96. # Try messing with resource limits to trigger errors and get
  97. # more coverage. Here, make it so we can only create files 1
  98. # byte in size, which will trigger an IOError in the server when
  99. # we create a table.
  100. limit = resource.getrlimit(resource.RLIMIT_FSIZE)
  101. resource.setrlimit(resource.RLIMIT_FSIZE, (1, limit[1]))
  102. with assert_raises(ServerError) as e:
  103. client.stream_create("/newton/hello", "RawData")
  104. resource.setrlimit(resource.RLIMIT_FSIZE, limit)
  105. client.close()
  106. def test_client_03_metadata(self):
  107. client = nilmdb.Client(url = testurl)
  108. # Set / get metadata
  109. eq_(client.stream_get_metadata("/newton/prep"), {})
  110. eq_(client.stream_get_metadata("/newton/raw"), {})
  111. meta1 = { "description": "The Data",
  112. "v_scale": "1.234" }
  113. meta2 = { "description": "The Data" }
  114. meta3 = { "v_scale": "1.234" }
  115. client.stream_set_metadata("/newton/prep", meta1)
  116. client.stream_update_metadata("/newton/prep", {})
  117. client.stream_update_metadata("/newton/raw", meta2)
  118. client.stream_update_metadata("/newton/raw", meta3)
  119. eq_(client.stream_get_metadata("/newton/prep"), meta1)
  120. eq_(client.stream_get_metadata("/newton/raw"), meta1)
  121. eq_(client.stream_get_metadata("/newton/raw",
  122. [ "description" ] ), meta2)
  123. eq_(client.stream_get_metadata("/newton/raw",
  124. [ "description", "v_scale" ] ), meta1)
  125. # missing key
  126. eq_(client.stream_get_metadata("/newton/raw", "descr"),
  127. { "descr": None })
  128. eq_(client.stream_get_metadata("/newton/raw", [ "descr" ]),
  129. { "descr": None })
  130. # test wrong types (list instead of dict)
  131. with assert_raises(ClientError):
  132. client.stream_set_metadata("/newton/prep", [1,2,3])
  133. with assert_raises(ClientError):
  134. client.stream_update_metadata("/newton/prep", [1,2,3])
  135. client.close()
  136. def test_client_04_insert(self):
  137. client = nilmdb.Client(url = testurl)
  138. datetime_tz.localtz_set("America/New_York")
  139. testfile = "tests/data/prep-20120323T1000"
  140. start = datetime_tz.datetime_tz.smartparse("20120323T1000")
  141. start = start.totimestamp()
  142. rate = 120
  143. # First try a nonexistent path
  144. data = timestamper.TimestamperRate(testfile, start, 120)
  145. with assert_raises(ClientError) as e:
  146. result = client.stream_insert("/newton/no-such-path", data)
  147. in_("404 Not Found", str(e.exception))
  148. # Now try reversed timestamps
  149. data = timestamper.TimestamperRate(testfile, start, 120)
  150. data = reversed(list(data))
  151. with assert_raises(ClientError) as e:
  152. result = client.stream_insert("/newton/prep", data)
  153. in_("400 Bad Request", str(e.exception))
  154. in_("timestamp is not monotonically increasing", str(e.exception))
  155. # Now try empty data (no server request made)
  156. empty = cStringIO.StringIO("")
  157. data = timestamper.TimestamperRate(empty, start, 120)
  158. result = client.stream_insert("/newton/prep", data)
  159. eq_(result, None)
  160. # Try forcing a server request with empty data
  161. with assert_raises(ClientError) as e:
  162. client.http.put("stream/insert", "", { "path": "/newton/prep",
  163. "start": 0, "end": 0 })
  164. in_("400 Bad Request", str(e.exception))
  165. in_("no data provided", str(e.exception))
  166. # Specify start/end (starts too late)
  167. data = timestamper.TimestamperRate(testfile, start, 120)
  168. with assert_raises(ClientError) as e:
  169. result = client.stream_insert("/newton/prep", data,
  170. start + 5, start + 120)
  171. in_("400 Bad Request", str(e.exception))
  172. in_("Data timestamp 1332511200.0 < start time 1332511205.0",
  173. str(e.exception))
  174. # Specify start/end (ends too early)
  175. data = timestamper.TimestamperRate(testfile, start, 120)
  176. with assert_raises(ClientError) as e:
  177. result = client.stream_insert("/newton/prep", data,
  178. start, start + 1)
  179. in_("400 Bad Request", str(e.exception))
  180. # Client chunks the input, so the exact timestamp here might change
  181. # if the chunk positions change.
  182. in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
  183. str(e.exception))
  184. # Now do the real load
  185. data = timestamper.TimestamperRate(testfile, start, 120)
  186. result = client.stream_insert("/newton/prep", data,
  187. start, start + 119.999777)
  188. eq_(result, "ok")
  189. # Verify the intervals. Should be just one, even if the data
  190. # was inserted in chunks, due to nilmdb interval concatenation.
  191. intervals = list(client.stream_intervals("/newton/prep"))
  192. eq_(intervals, [[start, start + 119.999777]])
  193. # Try some overlapping data -- just insert it again
  194. data = timestamper.TimestamperRate(testfile, start, 120)
  195. with assert_raises(ClientError) as e:
  196. result = client.stream_insert("/newton/prep", data)
  197. in_("400 Bad Request", str(e.exception))
  198. in_("verlap", str(e.exception))
  199. client.close()
  200. def test_client_05_extractremove(self):
  201. # Misc tests for extract and remove. Most of them are in test_cmdline.
  202. client = nilmdb.Client(url = testurl)
  203. for x in client.stream_extract("/newton/prep", 123, 123):
  204. raise AssertionError("shouldn't be any data for this request")
  205. with assert_raises(ClientError) as e:
  206. client.stream_remove("/newton/prep", 123, 120)
  207. client.close()
  208. def test_client_06_generators(self):
  209. # A lot of the client functionality is already tested by test_cmdline,
  210. # but this gets a bit more coverage that cmdline misses.
  211. client = nilmdb.Client(url = testurl)
  212. # Trigger a client error in generator
  213. start = datetime_tz.datetime_tz.smartparse("20120323T2000")
  214. end = datetime_tz.datetime_tz.smartparse("20120323T1000")
  215. for function in [ client.stream_intervals, client.stream_extract ]:
  216. with assert_raises(ClientError) as e:
  217. function("/newton/prep",
  218. start.totimestamp(),
  219. end.totimestamp()).next()
  220. in_("400 Bad Request", str(e.exception))
  221. in_("end before start", str(e.exception))
  222. # Trigger a curl error in generator
  223. with assert_raises(ServerError) as e:
  224. client.http.get_gen("http://nosuchurl/").next()
  225. # Trigger a curl error in generator
  226. with assert_raises(ServerError) as e:
  227. client.http.get_gen("http://nosuchurl/").next()
  228. # Check non-json version of string output
  229. eq_(json.loads(client.http.get("/stream/list",retjson=False)),
  230. client.http.get("/stream/list",retjson=True))
  231. # Check non-json version of generator output
  232. for (a, b) in itertools.izip(
  233. client.http.get_gen("/stream/list",retjson=False),
  234. client.http.get_gen("/stream/list",retjson=True)):
  235. eq_(json.loads(a), b)
  236. # Check PUT with generator out
  237. with assert_raises(ClientError) as e:
  238. client.http.put_gen("stream/insert", "",
  239. { "path": "/newton/prep",
  240. "start": 0, "end": 0 }).next()
  241. in_("400 Bad Request", str(e.exception))
  242. in_("no data provided", str(e.exception))
  243. # Check 404 for missing streams
  244. for function in [ client.stream_intervals, client.stream_extract ]:
  245. with assert_raises(ClientError) as e:
  246. function("/no/such/stream").next()
  247. in_("404 Not Found", str(e.exception))
  248. in_("No such stream", str(e.exception))
  249. client.close()
  250. def test_client_07_headers(self):
  251. # Make sure that /stream/intervals and /stream/extract
  252. # properly return streaming, chunked, text/plain response.
  253. # Pokes around in client.http internals a bit to look at the
  254. # response headers.
  255. client = nilmdb.Client(url = testurl)
  256. http = client.http
  257. # Use a warning rather than returning a test failure, so that we can
  258. # still disable chunked responses for debugging.
  259. # Intervals
  260. x = http.get("stream/intervals", { "path": "/newton/prep" },
  261. retjson=False)
  262. lines_(x, 1)
  263. if "Transfer-Encoding: chunked" not in http._headers:
  264. warnings.warn("Non-chunked HTTP response for /stream/intervals")
  265. if "Content-Type: text/plain;charset=utf-8" not in http._headers:
  266. raise AssertionError("/stream/intervals is not text/plain:\n" +
  267. http._headers)
  268. # Extract
  269. x = http.get("stream/extract",
  270. { "path": "/newton/prep",
  271. "start": "123",
  272. "end": "123" }, retjson=False)
  273. if "Transfer-Encoding: chunked" not in http._headers:
  274. warnings.warn("Non-chunked HTTP response for /stream/extract")
  275. if "Content-Type: text/plain;charset=utf-8" not in http._headers:
  276. raise AssertionError("/stream/extract is not text/plain:\n" +
  277. http._headers)
  278. # Make sure Access-Control-Allow-Origin gets set
  279. if "Access-Control-Allow-Origin: " not in http._headers:
  280. raise AssertionError("No Access-Control-Allow-Origin (CORS) "
  281. "header in /stream/extract response:\n" +
  282. http._headers)
  283. client.close()
  284. def test_client_08_unicode(self):
  285. # Basic Unicode tests
  286. client = nilmdb.Client(url = testurl)
  287. # Delete streams that exist
  288. for stream in client.stream_list():
  289. client.stream_destroy(stream[0])
  290. # Database is empty
  291. eq_(client.stream_list(), [])
  292. # Create Unicode stream, match it
  293. raw = [ u"/düsseldorf/raw", u"uint16_6" ]
  294. prep = [ u"/düsseldorf/prep", u"uint16_6" ]
  295. client.stream_create(*raw)
  296. eq_(client.stream_list(), [raw])
  297. eq_(client.stream_list(layout=raw[1]), [raw])
  298. eq_(client.stream_list(path=raw[0]), [raw])
  299. client.stream_create(*prep)
  300. eq_(client.stream_list(), [prep, raw])
  301. # Set / get metadata with Unicode keys and values
  302. eq_(client.stream_get_metadata(raw[0]), {})
  303. eq_(client.stream_get_metadata(prep[0]), {})
  304. meta1 = { u"alpha": u"α",
  305. u"β": u"beta" }
  306. meta2 = { u"alpha": u"α" }
  307. meta3 = { u"β": u"beta" }
  308. client.stream_set_metadata(prep[0], meta1)
  309. client.stream_update_metadata(prep[0], {})
  310. client.stream_update_metadata(raw[0], meta2)
  311. client.stream_update_metadata(raw[0], meta3)
  312. eq_(client.stream_get_metadata(prep[0]), meta1)
  313. eq_(client.stream_get_metadata(raw[0]), meta1)
  314. eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
  315. eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
  316. client.close()
  317. def test_client_09_closing(self):
  318. # Make sure we actually close sockets correctly. New
  319. # connections will block for a while if they're not, since the
  320. # server will stop accepting new connections.
  321. for test in [1, 2]:
  322. start = time.time()
  323. for i in range(50):
  324. if time.time() - start > 15:
  325. raise AssertionError("Connections seem to be blocking... "
  326. "probably not closing properly.")
  327. if test == 1:
  328. # explicit close
  329. client = nilmdb.Client(url = testurl)
  330. with assert_raises(ClientError) as e:
  331. client.stream_remove("/newton/prep", 123, 120)
  332. client.close() # remove this to see the failure
  333. elif test == 2:
  334. # use the context manager
  335. with nilmdb.Client(url = testurl) as c:
  336. with assert_raises(ClientError) as e:
  337. c.stream_remove("/newton/prep", 123, 120)
  338. def test_client_10_context(self):
  339. # Test using the client's stream insertion context manager to
  340. # insert data.
  341. client = nilmdb.Client(testurl)
  342. client.stream_create("/context/test", "uint16_1")
  343. with client.stream_insert_context("/context/test") as ctx:
  344. # override _max_data to trigger frequent server updates
  345. ctx._max_data = 15
  346. with assert_raises(ValueError):
  347. ctx.insert_line("100 1")
  348. ctx.insert_line("100 1\n")
  349. ctx.insert_iter([ "101 1\n",
  350. "102 1\n",
  351. "103 1\n" ])
  352. ctx.insert_line("104 1\n")
  353. ctx.insert_line("105 1\n")
  354. ctx.finalize()
  355. ctx.insert_line("106 1\n")
  356. ctx.update_end(106.5)
  357. ctx.finalize()
  358. ctx.update_start(106.8)
  359. ctx.insert_line("107 1\n")
  360. ctx.insert_line("108 1\n")
  361. ctx.insert_line("109 1\n")
  362. ctx.insert_line("110 1\n")
  363. ctx.insert_line("111 1\n")
  364. ctx.update_end(113)
  365. ctx.insert_line("112 1\n")
  366. ctx.update_end(114)
  367. ctx.insert_line("113 1\n")
  368. ctx.update_end(115)
  369. ctx.insert_line("114 1\n")
  370. ctx.finalize()
  371. with assert_raises(ClientError):
  372. with client.stream_insert_context("/context/test", 100, 200) as ctx:
  373. ctx.insert_line("115 1\n")
  374. with assert_raises(ClientError):
  375. with client.stream_insert_context("/context/test", 200, 300) as ctx:
  376. ctx.insert_line("115 1\n")
  377. with client.stream_insert_context("/context/test", 200, 300) as ctx:
  378. # make sure our override wasn't permanent
  379. ne_(ctx._max_data, 15)
  380. ctx.insert_line("225 1\n")
  381. ctx.finalize()
  382. eq_(list(client.stream_intervals("/context/test")),
  383. [ [ 100, 105.000001 ],
  384. [ 106, 106.5 ],
  385. [ 106.8, 115 ],
  386. [ 200, 300 ] ])
  387. client.stream_destroy("/context/test")
  388. client.close()