You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

402 lines
16 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb
  3. from nilmdb.utils.printf import *
  4. from nilmdb.utils import timestamper
  5. from nilmdb.client import ClientError, ServerError
  6. from nilmdb.utils import datetime_tz
  7. from nose.tools import *
  8. from nose.tools import assert_raises
  9. import itertools
  10. import distutils.version
  11. import os
  12. import sys
  13. import threading
  14. import cStringIO
  15. import simplejson as json
  16. import unittest
  17. import warnings
  18. import resource
  19. import time
  20. from testutil.helpers import *
  21. testdb = "tests/client-testdb"
  22. def setup_module():
  23. global test_server, test_db
  24. # Clear out DB
  25. recursive_unlink(testdb)
  26. # Start web app on a custom port
  27. test_db = nilmdb.NilmDB(testdb, sync = False)
  28. test_server = nilmdb.Server(test_db, host = "127.0.0.1",
  29. port = 12380, stoppable = False,
  30. fast_shutdown = True,
  31. force_traceback = False)
  32. test_server.start(blocking = False)
  33. def teardown_module():
  34. global test_server, test_db
  35. # Close web app
  36. test_server.stop()
  37. test_db.close()
  38. class TestClient(object):
  39. def test_client_1_basic(self):
  40. # Test a fake host
  41. client = nilmdb.Client(url = "http://localhost:1/")
  42. with assert_raises(nilmdb.client.ServerError):
  43. client.version()
  44. client.close()
  45. # Trigger same error with a PUT request
  46. client = nilmdb.Client(url = "http://localhost:1/")
  47. with assert_raises(nilmdb.client.ServerError):
  48. client.version()
  49. client.close()
  50. # Then a fake URL on a real host
  51. client = nilmdb.Client(url = "http://localhost:12380/fake/")
  52. with assert_raises(nilmdb.client.ClientError):
  53. client.version()
  54. client.close()
  55. # Now a real URL with no http:// prefix
  56. client = nilmdb.Client(url = "localhost:12380")
  57. version = client.version()
  58. client.close()
  59. # Now use the real URL
  60. client = nilmdb.Client(url = "http://localhost:12380/")
  61. version = client.version()
  62. eq_(distutils.version.LooseVersion(version),
  63. distutils.version.LooseVersion(test_server.version))
  64. # Bad URLs should give 404, not 500
  65. with assert_raises(ClientError):
  66. client.http.get("/stream/create")
  67. client.close()
  68. def test_client_2_createlist(self):
  69. # Basic stream tests, like those in test_nilmdb:test_stream
  70. client = nilmdb.Client(url = "http://localhost:12380/")
  71. # Database starts empty
  72. eq_(client.stream_list(), [])
  73. # Bad path
  74. with assert_raises(ClientError):
  75. client.stream_create("foo/bar/baz", "PrepData")
  76. with assert_raises(ClientError):
  77. client.stream_create("/foo", "PrepData")
  78. # Bad layout type
  79. with assert_raises(ClientError):
  80. client.stream_create("/newton/prep", "NoSuchLayout")
  81. # Create three streams
  82. client.stream_create("/newton/prep", "PrepData")
  83. client.stream_create("/newton/raw", "RawData")
  84. client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
  85. # Verify we got 3 streams
  86. eq_(client.stream_list(), [ ["/newton/prep", "PrepData"],
  87. ["/newton/raw", "RawData"],
  88. ["/newton/zzz/rawnotch", "RawNotchedData"]
  89. ])
  90. # Match just one type or one path
  91. eq_(client.stream_list(layout="RawData"),
  92. [ ["/newton/raw", "RawData"] ])
  93. eq_(client.stream_list(path="/newton/raw"),
  94. [ ["/newton/raw", "RawData"] ])
  95. # Try messing with resource limits to trigger errors and get
  96. # more coverage. Here, make it so we can only create files 1
  97. # byte in size, which will trigger an IOError in the server when
  98. # we create a table.
  99. limit = resource.getrlimit(resource.RLIMIT_FSIZE)
  100. resource.setrlimit(resource.RLIMIT_FSIZE, (1, limit[1]))
  101. with assert_raises(ServerError) as e:
  102. client.stream_create("/newton/hello", "RawData")
  103. resource.setrlimit(resource.RLIMIT_FSIZE, limit)
  104. client.close()
  105. def test_client_3_metadata(self):
  106. client = nilmdb.Client(url = "http://localhost:12380/")
  107. # Set / get metadata
  108. eq_(client.stream_get_metadata("/newton/prep"), {})
  109. eq_(client.stream_get_metadata("/newton/raw"), {})
  110. meta1 = { "description": "The Data",
  111. "v_scale": "1.234" }
  112. meta2 = { "description": "The Data" }
  113. meta3 = { "v_scale": "1.234" }
  114. client.stream_set_metadata("/newton/prep", meta1)
  115. client.stream_update_metadata("/newton/prep", {})
  116. client.stream_update_metadata("/newton/raw", meta2)
  117. client.stream_update_metadata("/newton/raw", meta3)
  118. eq_(client.stream_get_metadata("/newton/prep"), meta1)
  119. eq_(client.stream_get_metadata("/newton/raw"), meta1)
  120. eq_(client.stream_get_metadata("/newton/raw",
  121. [ "description" ] ), meta2)
  122. eq_(client.stream_get_metadata("/newton/raw",
  123. [ "description", "v_scale" ] ), meta1)
  124. # missing key
  125. eq_(client.stream_get_metadata("/newton/raw", "descr"),
  126. { "descr": None })
  127. eq_(client.stream_get_metadata("/newton/raw", [ "descr" ]),
  128. { "descr": None })
  129. # test wrong types (list instead of dict)
  130. with assert_raises(ClientError):
  131. client.stream_set_metadata("/newton/prep", [1,2,3])
  132. with assert_raises(ClientError):
  133. client.stream_update_metadata("/newton/prep", [1,2,3])
  134. client.close()
  135. def test_client_4_insert(self):
  136. client = nilmdb.Client(url = "http://localhost:12380/")
  137. datetime_tz.localtz_set("America/New_York")
  138. testfile = "tests/data/prep-20120323T1000"
  139. start = datetime_tz.datetime_tz.smartparse("20120323T1000")
  140. start = start.totimestamp()
  141. rate = 120
  142. # First try a nonexistent path
  143. data = timestamper.TimestamperRate(testfile, start, 120)
  144. with assert_raises(ClientError) as e:
  145. result = client.stream_insert("/newton/no-such-path", data)
  146. in_("404 Not Found", str(e.exception))
  147. # Now try reversed timestamps
  148. data = timestamper.TimestamperRate(testfile, start, 120)
  149. data = reversed(list(data))
  150. with assert_raises(ClientError) as e:
  151. result = client.stream_insert("/newton/prep", data)
  152. in_("400 Bad Request", str(e.exception))
  153. in_("timestamp is not monotonically increasing", str(e.exception))
  154. # Now try empty data (no server request made)
  155. empty = cStringIO.StringIO("")
  156. data = timestamper.TimestamperRate(empty, start, 120)
  157. result = client.stream_insert("/newton/prep", data)
  158. eq_(result, None)
  159. # Try forcing a server request with empty data
  160. with assert_raises(ClientError) as e:
  161. client.http.put("stream/insert", "", { "path": "/newton/prep",
  162. "start": 0, "end": 0 })
  163. in_("400 Bad Request", str(e.exception))
  164. in_("no data provided", str(e.exception))
  165. # Specify start/end (starts too late)
  166. data = timestamper.TimestamperRate(testfile, start, 120)
  167. with assert_raises(ClientError) as e:
  168. result = client.stream_insert("/newton/prep", data,
  169. start + 5, start + 120)
  170. in_("400 Bad Request", str(e.exception))
  171. in_("Data timestamp 1332511200.0 < start time 1332511205.0",
  172. str(e.exception))
  173. # Specify start/end (ends too early)
  174. data = timestamper.TimestamperRate(testfile, start, 120)
  175. with assert_raises(ClientError) as e:
  176. result = client.stream_insert("/newton/prep", data,
  177. start, start + 1)
  178. in_("400 Bad Request", str(e.exception))
  179. # Client chunks the input, so the exact timestamp here might change
  180. # if the chunk positions change.
  181. in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
  182. str(e.exception))
  183. # Now do the real load
  184. data = timestamper.TimestamperRate(testfile, start, 120)
  185. result = client.stream_insert("/newton/prep", data,
  186. start, start + 119.999777)
  187. eq_(result, "ok")
  188. # Verify the intervals. Should be just one, even if the data
  189. # was inserted in chunks, due to nilmdb interval concatenation.
  190. intervals = list(client.stream_intervals("/newton/prep"))
  191. eq_(intervals, [[start, start + 119.999777]])
  192. # Try some overlapping data -- just insert it again
  193. data = timestamper.TimestamperRate(testfile, start, 120)
  194. with assert_raises(ClientError) as e:
  195. result = client.stream_insert("/newton/prep", data)
  196. in_("400 Bad Request", str(e.exception))
  197. in_("verlap", str(e.exception))
  198. client.close()
  199. def test_client_5_extractremove(self):
  200. # Misc tests for extract and remove. Most of them are in test_cmdline.
  201. client = nilmdb.Client(url = "http://localhost:12380/")
  202. for x in client.stream_extract("/newton/prep", 123, 123):
  203. raise AssertionError("shouldn't be any data for this request")
  204. with assert_raises(ClientError) as e:
  205. client.stream_remove("/newton/prep", 123, 120)
  206. client.close()
  207. def test_client_6_generators(self):
  208. # A lot of the client functionality is already tested by test_cmdline,
  209. # but this gets a bit more coverage that cmdline misses.
  210. client = nilmdb.Client(url = "http://localhost:12380/")
  211. # Trigger a client error in generator
  212. start = datetime_tz.datetime_tz.smartparse("20120323T2000")
  213. end = datetime_tz.datetime_tz.smartparse("20120323T1000")
  214. for function in [ client.stream_intervals, client.stream_extract ]:
  215. with assert_raises(ClientError) as e:
  216. function("/newton/prep",
  217. start.totimestamp(),
  218. end.totimestamp()).next()
  219. in_("400 Bad Request", str(e.exception))
  220. in_("end before start", str(e.exception))
  221. # Trigger a curl error in generator
  222. with assert_raises(ServerError) as e:
  223. client.http.get_gen("http://nosuchurl/").next()
  224. # Trigger a curl error in generator
  225. with assert_raises(ServerError) as e:
  226. client.http.get_gen("http://nosuchurl/").next()
  227. # Check non-json version of string output
  228. eq_(json.loads(client.http.get("/stream/list",retjson=False)),
  229. client.http.get("/stream/list",retjson=True))
  230. # Check non-json version of generator output
  231. for (a, b) in itertools.izip(
  232. client.http.get_gen("/stream/list",retjson=False),
  233. client.http.get_gen("/stream/list",retjson=True)):
  234. eq_(json.loads(a), b)
  235. # Check PUT with generator out
  236. with assert_raises(ClientError) as e:
  237. client.http.put_gen("stream/insert", "",
  238. { "path": "/newton/prep",
  239. "start": 0, "end": 0 }).next()
  240. in_("400 Bad Request", str(e.exception))
  241. in_("no data provided", str(e.exception))
  242. # Check 404 for missing streams
  243. for function in [ client.stream_intervals, client.stream_extract ]:
  244. with assert_raises(ClientError) as e:
  245. function("/no/such/stream").next()
  246. in_("404 Not Found", str(e.exception))
  247. in_("No such stream", str(e.exception))
  248. client.close()
  249. def test_client_7_headers(self):
  250. # Make sure that /stream/intervals and /stream/extract
  251. # properly return streaming, chunked, text/plain response.
  252. # Pokes around in client.http internals a bit to look at the
  253. # response headers.
  254. client = nilmdb.Client(url = "http://localhost:12380/")
  255. http = client.http
  256. # Use a warning rather than returning a test failure, so that we can
  257. # still disable chunked responses for debugging.
  258. # Intervals
  259. x = http.get("stream/intervals", { "path": "/newton/prep" },
  260. retjson=False)
  261. lines_(x, 1)
  262. if "Transfer-Encoding: chunked" not in http._headers:
  263. warnings.warn("Non-chunked HTTP response for /stream/intervals")
  264. if "Content-Type: text/plain;charset=utf-8" not in http._headers:
  265. raise AssertionError("/stream/intervals is not text/plain:\n" +
  266. http._headers)
  267. # Extract
  268. x = http.get("stream/extract",
  269. { "path": "/newton/prep",
  270. "start": "123",
  271. "end": "123" }, retjson=False)
  272. if "Transfer-Encoding: chunked" not in http._headers:
  273. warnings.warn("Non-chunked HTTP response for /stream/extract")
  274. if "Content-Type: text/plain;charset=utf-8" not in http._headers:
  275. raise AssertionError("/stream/extract is not text/plain:\n" +
  276. http._headers)
  277. # Make sure Access-Control-Allow-Origin gets set
  278. if "Access-Control-Allow-Origin: " not in http._headers:
  279. raise AssertionError("No Access-Control-Allow-Origin (CORS) "
  280. "header in /stream/extract response:\n" +
  281. http._headers)
  282. client.close()
  283. def test_client_8_unicode(self):
  284. # Basic Unicode tests
  285. client = nilmdb.Client(url = "http://localhost:12380/")
  286. # Delete streams that exist
  287. for stream in client.stream_list():
  288. client.stream_destroy(stream[0])
  289. # Database is empty
  290. eq_(client.stream_list(), [])
  291. # Create Unicode stream, match it
  292. raw = [ u"/düsseldorf/raw", u"uint16_6" ]
  293. prep = [ u"/düsseldorf/prep", u"uint16_6" ]
  294. client.stream_create(*raw)
  295. eq_(client.stream_list(), [raw])
  296. eq_(client.stream_list(layout=raw[1]), [raw])
  297. eq_(client.stream_list(path=raw[0]), [raw])
  298. client.stream_create(*prep)
  299. eq_(client.stream_list(), [prep, raw])
  300. # Set / get metadata with Unicode keys and values
  301. eq_(client.stream_get_metadata(raw[0]), {})
  302. eq_(client.stream_get_metadata(prep[0]), {})
  303. meta1 = { u"alpha": u"α",
  304. u"β": u"beta" }
  305. meta2 = { u"alpha": u"α" }
  306. meta3 = { u"β": u"beta" }
  307. client.stream_set_metadata(prep[0], meta1)
  308. client.stream_update_metadata(prep[0], {})
  309. client.stream_update_metadata(raw[0], meta2)
  310. client.stream_update_metadata(raw[0], meta3)
  311. eq_(client.stream_get_metadata(prep[0]), meta1)
  312. eq_(client.stream_get_metadata(raw[0]), meta1)
  313. eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
  314. eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
  315. client.close()
  316. def test_client_9_closing(self):
  317. # Make sure we actually close sockets correctly. New
  318. # connections will block for a while if they're not, since the
  319. # server will stop accepting new connections.
  320. for test in [1, 2]:
  321. start = time.time()
  322. for i in range(50):
  323. if time.time() - start > 15:
  324. raise AssertionError("Connections seem to be blocking... "
  325. "probably not closing properly.")
  326. if test == 1:
  327. # explicit close
  328. client = nilmdb.Client(url = "http://localhost:12380/")
  329. with assert_raises(ClientError) as e:
  330. client.stream_remove("/newton/prep", 123, 120)
  331. client.close() # remove this to see the failure
  332. elif test == 2:
  333. # use the context manager
  334. with nilmdb.Client(url = "http://localhost:12380/") as c:
  335. with assert_raises(ClientError) as e:
  336. c.stream_remove("/newton/prep", 123, 120)