You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

385 lines
16 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb
  3. from nilmdb.utils.printf import *
  4. from nilmdb.utils import timestamper
  5. from nilmdb.client import ClientError, ServerError
  6. from nilmdb.utils import datetime_tz
  7. from nose.tools import *
  8. from nose.tools import assert_raises
  9. import itertools
  10. import distutils.version
  11. import os
  12. import sys
  13. import threading
  14. import cStringIO
  15. import simplejson as json
  16. import unittest
  17. import warnings
  18. import resource
  19. import time
  20. from testutil.helpers import *
  21. testdb = "tests/client-testdb"
  22. def setup_module():
  23. global test_server, test_db
  24. # Clear out DB
  25. recursive_unlink(testdb)
  26. # Start web app on a custom port
  27. test_db = nilmdb.NilmDB(testdb, sync = False)
  28. test_server = nilmdb.Server(test_db, host = "127.0.0.1",
  29. port = 12380, stoppable = False,
  30. fast_shutdown = True,
  31. force_traceback = False)
  32. test_server.start(blocking = False)
  33. def teardown_module():
  34. global test_server, test_db
  35. # Close web app
  36. test_server.stop()
  37. test_db.close()
  38. class TestClient(object):
  39. def test_client_1_basic(self):
  40. # Test a fake host
  41. client = nilmdb.Client(url = "http://localhost:1/")
  42. with assert_raises(nilmdb.client.ServerError):
  43. client.version()
  44. # Trigger same error with a PUT request
  45. client = nilmdb.Client(url = "http://localhost:1/")
  46. with assert_raises(nilmdb.client.ServerError):
  47. client.version()
  48. # Then a fake URL on a real host
  49. client = nilmdb.Client(url = "http://localhost:12380/fake/")
  50. with assert_raises(nilmdb.client.ClientError):
  51. client.version()
  52. # Now a real URL with no http:// prefix
  53. client = nilmdb.Client(url = "localhost:12380")
  54. version = client.version()
  55. # Now use the real URL
  56. client = nilmdb.Client(url = "http://localhost:12380/")
  57. version = client.version()
  58. eq_(distutils.version.LooseVersion(version),
  59. distutils.version.LooseVersion(test_server.version))
  60. # Bad URLs should give 404, not 500
  61. with assert_raises(ClientError):
  62. client.http.get("/stream/create")
  63. def test_client_2_createlist(self):
  64. # Basic stream tests, like those in test_nilmdb:test_stream
  65. client = nilmdb.Client(url = "http://localhost:12380/")
  66. # Database starts empty
  67. eq_(client.stream_list(), [])
  68. # Bad path
  69. with assert_raises(ClientError):
  70. client.stream_create("foo/bar/baz", "PrepData")
  71. with assert_raises(ClientError):
  72. client.stream_create("/foo", "PrepData")
  73. # Bad layout type
  74. with assert_raises(ClientError):
  75. client.stream_create("/newton/prep", "NoSuchLayout")
  76. # Create three streams
  77. client.stream_create("/newton/prep", "PrepData")
  78. client.stream_create("/newton/raw", "RawData")
  79. client.stream_create("/newton/zzz/rawnotch", "RawNotchedData")
  80. # Verify we got 3 streams
  81. eq_(client.stream_list(), [ ["/newton/prep", "PrepData"],
  82. ["/newton/raw", "RawData"],
  83. ["/newton/zzz/rawnotch", "RawNotchedData"]
  84. ])
  85. # Match just one type or one path
  86. eq_(client.stream_list(layout="RawData"),
  87. [ ["/newton/raw", "RawData"] ])
  88. eq_(client.stream_list(path="/newton/raw"),
  89. [ ["/newton/raw", "RawData"] ])
  90. # Try messing with resource limits to trigger errors and get
  91. # more coverage. Here, make it so we can only create files 1
  92. # byte in size, which will trigger an IOError in the server when
  93. # we create a table.
  94. limit = resource.getrlimit(resource.RLIMIT_FSIZE)
  95. resource.setrlimit(resource.RLIMIT_FSIZE, (1, limit[1]))
  96. with assert_raises(ServerError) as e:
  97. client.stream_create("/newton/hello", "RawData")
  98. resource.setrlimit(resource.RLIMIT_FSIZE, limit)
  99. def test_client_3_metadata(self):
  100. client = nilmdb.Client(url = "http://localhost:12380/")
  101. # Set / get metadata
  102. eq_(client.stream_get_metadata("/newton/prep"), {})
  103. eq_(client.stream_get_metadata("/newton/raw"), {})
  104. meta1 = { "description": "The Data",
  105. "v_scale": "1.234" }
  106. meta2 = { "description": "The Data" }
  107. meta3 = { "v_scale": "1.234" }
  108. client.stream_set_metadata("/newton/prep", meta1)
  109. client.stream_update_metadata("/newton/prep", {})
  110. client.stream_update_metadata("/newton/raw", meta2)
  111. client.stream_update_metadata("/newton/raw", meta3)
  112. eq_(client.stream_get_metadata("/newton/prep"), meta1)
  113. eq_(client.stream_get_metadata("/newton/raw"), meta1)
  114. eq_(client.stream_get_metadata("/newton/raw",
  115. [ "description" ] ), meta2)
  116. eq_(client.stream_get_metadata("/newton/raw",
  117. [ "description", "v_scale" ] ), meta1)
  118. # missing key
  119. eq_(client.stream_get_metadata("/newton/raw", "descr"),
  120. { "descr": None })
  121. eq_(client.stream_get_metadata("/newton/raw", [ "descr" ]),
  122. { "descr": None })
  123. # test wrong types (list instead of dict)
  124. with assert_raises(ClientError):
  125. client.stream_set_metadata("/newton/prep", [1,2,3])
  126. with assert_raises(ClientError):
  127. client.stream_update_metadata("/newton/prep", [1,2,3])
  128. def test_client_4_insert(self):
  129. client = nilmdb.Client(url = "http://localhost:12380/")
  130. datetime_tz.localtz_set("America/New_York")
  131. testfile = "tests/data/prep-20120323T1000"
  132. start = datetime_tz.datetime_tz.smartparse("20120323T1000")
  133. start = start.totimestamp()
  134. rate = 120
  135. # First try a nonexistent path
  136. data = timestamper.TimestamperRate(testfile, start, 120)
  137. with assert_raises(ClientError) as e:
  138. result = client.stream_insert("/newton/no-such-path", data)
  139. in_("404 Not Found", str(e.exception))
  140. # Now try reversed timestamps
  141. data = timestamper.TimestamperRate(testfile, start, 120)
  142. data = reversed(list(data))
  143. with assert_raises(ClientError) as e:
  144. result = client.stream_insert("/newton/prep", data)
  145. in_("400 Bad Request", str(e.exception))
  146. in_("timestamp is not monotonically increasing", str(e.exception))
  147. # Now try empty data (no server request made)
  148. empty = cStringIO.StringIO("")
  149. data = timestamper.TimestamperRate(empty, start, 120)
  150. result = client.stream_insert("/newton/prep", data)
  151. eq_(result, None)
  152. # Try forcing a server request with empty data
  153. with assert_raises(ClientError) as e:
  154. client.http.put("stream/insert", "", { "path": "/newton/prep",
  155. "start": 0, "end": 0 })
  156. in_("400 Bad Request", str(e.exception))
  157. in_("no data provided", str(e.exception))
  158. # Specify start/end (starts too late)
  159. data = timestamper.TimestamperRate(testfile, start, 120)
  160. with assert_raises(ClientError) as e:
  161. result = client.stream_insert("/newton/prep", data,
  162. start + 5, start + 120)
  163. in_("400 Bad Request", str(e.exception))
  164. in_("Data timestamp 1332511200.0 < start time 1332511205.0",
  165. str(e.exception))
  166. # Specify start/end (ends too early)
  167. data = timestamper.TimestamperRate(testfile, start, 120)
  168. with assert_raises(ClientError) as e:
  169. result = client.stream_insert("/newton/prep", data,
  170. start, start + 1)
  171. in_("400 Bad Request", str(e.exception))
  172. # Client chunks the input, so the exact timestamp here might change
  173. # if the chunk positions change.
  174. in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
  175. str(e.exception))
  176. # Now do the real load
  177. data = timestamper.TimestamperRate(testfile, start, 120)
  178. result = client.stream_insert("/newton/prep", data,
  179. start, start + 119.999777)
  180. eq_(result, "ok")
  181. # Verify the intervals. Should be just one, even if the data
  182. # was inserted in chunks, due to nilmdb interval concatenation.
  183. intervals = list(client.stream_intervals("/newton/prep"))
  184. eq_(intervals, [[start, start + 119.999777]])
  185. # Try some overlapping data -- just insert it again
  186. data = timestamper.TimestamperRate(testfile, start, 120)
  187. with assert_raises(ClientError) as e:
  188. result = client.stream_insert("/newton/prep", data)
  189. in_("400 Bad Request", str(e.exception))
  190. in_("verlap", str(e.exception))
  191. def test_client_5_extractremove(self):
  192. # Misc tests for extract and remove. Most of them are in test_cmdline.
  193. client = nilmdb.Client(url = "http://localhost:12380/")
  194. for x in client.stream_extract("/newton/prep", 123, 123):
  195. raise AssertionError("shouldn't be any data for this request")
  196. with assert_raises(ClientError) as e:
  197. client.stream_remove("/newton/prep", 123, 120)
  198. def test_client_6_generators(self):
  199. # A lot of the client functionality is already tested by test_cmdline,
  200. # but this gets a bit more coverage that cmdline misses.
  201. client = nilmdb.Client(url = "http://localhost:12380/")
  202. # Trigger a client error in generator
  203. start = datetime_tz.datetime_tz.smartparse("20120323T2000")
  204. end = datetime_tz.datetime_tz.smartparse("20120323T1000")
  205. for function in [ client.stream_intervals, client.stream_extract ]:
  206. with assert_raises(ClientError) as e:
  207. function("/newton/prep",
  208. start.totimestamp(),
  209. end.totimestamp()).next()
  210. in_("400 Bad Request", str(e.exception))
  211. in_("end before start", str(e.exception))
  212. # Trigger a curl error in generator
  213. with assert_raises(ServerError) as e:
  214. client.http.get_gen("http://nosuchurl/").next()
  215. # Trigger a curl error in generator
  216. with assert_raises(ServerError) as e:
  217. client.http.get_gen("http://nosuchurl/").next()
  218. # Check non-json version of string output
  219. eq_(json.loads(client.http.get("/stream/list",retjson=False)),
  220. client.http.get("/stream/list",retjson=True))
  221. # Check non-json version of generator output
  222. for (a, b) in itertools.izip(
  223. client.http.get_gen("/stream/list",retjson=False),
  224. client.http.get_gen("/stream/list",retjson=True)):
  225. eq_(json.loads(a), b)
  226. # Check PUT with generator out
  227. with assert_raises(ClientError) as e:
  228. client.http.put_gen("stream/insert", "",
  229. { "path": "/newton/prep",
  230. "start": 0, "end": 0 }).next()
  231. in_("400 Bad Request", str(e.exception))
  232. in_("no data provided", str(e.exception))
  233. # Check 404 for missing streams
  234. for function in [ client.stream_intervals, client.stream_extract ]:
  235. with assert_raises(ClientError) as e:
  236. function("/no/such/stream").next()
  237. in_("404 Not Found", str(e.exception))
  238. in_("No such stream", str(e.exception))
  239. def test_client_7_headers(self):
  240. # Make sure that /stream/intervals and /stream/extract
  241. # properly return streaming, chunked, text/plain response.
  242. # Pokes around in client.http internals a bit to look at the
  243. # response headers.
  244. client = nilmdb.Client(url = "http://localhost:12380/")
  245. http = client.http
  246. # Use a warning rather than returning a test failure, so that we can
  247. # still disable chunked responses for debugging.
  248. # Intervals
  249. x = http.get("stream/intervals", { "path": "/newton/prep" },
  250. retjson=False)
  251. lines_(x, 1)
  252. if "Transfer-Encoding: chunked" not in http._headers:
  253. warnings.warn("Non-chunked HTTP response for /stream/intervals")
  254. if "Content-Type: text/plain;charset=utf-8" not in http._headers:
  255. raise AssertionError("/stream/intervals is not text/plain:\n" +
  256. http._headers)
  257. # Extract
  258. x = http.get("stream/extract",
  259. { "path": "/newton/prep",
  260. "start": "123",
  261. "end": "123" }, retjson=False)
  262. if "Transfer-Encoding: chunked" not in http._headers:
  263. warnings.warn("Non-chunked HTTP response for /stream/extract")
  264. if "Content-Type: text/plain;charset=utf-8" not in http._headers:
  265. raise AssertionError("/stream/extract is not text/plain:\n" +
  266. http._headers)
  267. # Make sure Access-Control-Allow-Origin gets set
  268. if "Access-Control-Allow-Origin: " not in http._headers:
  269. raise AssertionError("No Access-Control-Allow-Origin (CORS) "
  270. "header in /stream/extract response:\n" +
  271. http._headers)
  272. def test_client_8_unicode(self):
  273. # Basic Unicode tests
  274. client = nilmdb.Client(url = "http://localhost:12380/")
  275. # Delete streams that exist
  276. for stream in client.stream_list():
  277. client.stream_destroy(stream[0])
  278. # Database is empty
  279. eq_(client.stream_list(), [])
  280. # Create Unicode stream, match it
  281. raw = [ u"/düsseldorf/raw", u"uint16_6" ]
  282. prep = [ u"/düsseldorf/prep", u"uint16_6" ]
  283. client.stream_create(*raw)
  284. eq_(client.stream_list(), [raw])
  285. eq_(client.stream_list(layout=raw[1]), [raw])
  286. eq_(client.stream_list(path=raw[0]), [raw])
  287. client.stream_create(*prep)
  288. eq_(client.stream_list(), [prep, raw])
  289. # Set / get metadata with Unicode keys and values
  290. eq_(client.stream_get_metadata(raw[0]), {})
  291. eq_(client.stream_get_metadata(prep[0]), {})
  292. meta1 = { u"alpha": u"α",
  293. u"β": u"beta" }
  294. meta2 = { u"alpha": u"α" }
  295. meta3 = { u"β": u"beta" }
  296. client.stream_set_metadata(prep[0], meta1)
  297. client.stream_update_metadata(prep[0], {})
  298. client.stream_update_metadata(raw[0], meta2)
  299. client.stream_update_metadata(raw[0], meta3)
  300. eq_(client.stream_get_metadata(prep[0]), meta1)
  301. eq_(client.stream_get_metadata(raw[0]), meta1)
  302. eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2)
  303. eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1)
  304. def test_client_9_closing(self):
  305. # Make sure we actually close sockets correctly. New
  306. # connections will block for a while if they're not, since the
  307. # server will stop accepting new connections.
  308. for test in [1, 2]:
  309. start = time.time()
  310. for i in range(50):
  311. if time.time() - start > 15:
  312. raise AssertionError("Connections seem to be blocking... "
  313. "probably not closing properly.")
  314. if test == 1:
  315. # explicit close
  316. client = nilmdb.Client(url = "http://localhost:12380/")
  317. with assert_raises(ClientError) as e:
  318. client.stream_remove("/newton/prep", 123, 120)
  319. client.close() # remove this to see the failure
  320. elif test == 2:
  321. # use the context manager
  322. with nilmdb.Client(url = "http://localhost:12380/") as c:
  323. with assert_raises(ClientError) as e:
  324. c.stream_remove("/newton/prep", 123, 120)