You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

502 lines
18 KiB

  1. """CherryPy-based server for accessing NILM database via HTTP"""
  2. # Need absolute_import so that "import nilmdb" won't pull in
  3. # nilmdb.py, but will pull the nilmdb module instead.
  4. from __future__ import absolute_import
  5. import nilmdb
  6. from nilmdb.utils.printf import *
  7. from nilmdb.server.errors import *
  8. import cherrypy
  9. import sys
  10. import time
  11. import os
  12. import simplejson as json
  13. import decorator
  14. import traceback
  15. try:
  16. import cherrypy
  17. cherrypy.tools.json_out
  18. except: # pragma: no cover
  19. sys.stderr.write("Cherrypy 3.2+ required\n")
  20. sys.exit(1)
  21. class NilmApp(object):
  22. def __init__(self, db):
  23. self.db = db
  24. version = "1.2"
  25. # Decorators
  26. def chunked_response(func):
  27. """Decorator to enable chunked responses."""
  28. # Set this to False to get better tracebacks from some requests
  29. # (/stream/extract, /stream/intervals).
  30. func._cp_config = { 'response.stream': True }
  31. return func
  32. def response_type(content_type):
  33. """Return a decorator-generating function that sets the
  34. response type to the specified string."""
  35. def wrapper(func, *args, **kwargs):
  36. cherrypy.response.headers['Content-Type'] = content_type
  37. return func(*args, **kwargs)
  38. return decorator.decorator(wrapper)
  39. @decorator.decorator
  40. def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
  41. """Decorator to work around CherryPy bug #1200 in a response
  42. generator.
  43. Even if chunked responses are disabled, LookupError or
  44. UnicodeError exceptions may still be swallowed by CherryPy due to
  45. bug #1200. This throws them as generic Exceptions instead so that
  46. they make it through.
  47. """
  48. try:
  49. for val in func(*args, **kwargs):
  50. yield val
  51. except (LookupError, UnicodeError) as e:
  52. raise Exception("bug workaround; real exception is:\n" +
  53. traceback.format_exc())
  54. def exception_to_httperror(*expected):
  55. """Return a decorator-generating function that catches expected
  56. errors and throws a HTTPError describing it instead.
  57. @exception_to_httperror(NilmDBError, ValueError)
  58. def foo():
  59. pass
  60. """
  61. def wrapper(func, *args, **kwargs):
  62. try:
  63. return func(*args, **kwargs)
  64. except expected as e:
  65. message = sprintf("%s", str(e))
  66. raise cherrypy.HTTPError("400 Bad Request", message)
  67. # We need to preserve the function's argspecs for CherryPy to
  68. # handle argument errors correctly. Decorator.decorator takes
  69. # care of that.
  70. return decorator.decorator(wrapper)
  71. # CherryPy apps
  72. class Root(NilmApp):
  73. """Root application for NILM database"""
  74. def __init__(self, db, version):
  75. super(Root, self).__init__(db)
  76. self.server_version = version
  77. # /
  78. @cherrypy.expose
  79. def index(self):
  80. raise cherrypy.NotFound()
  81. # /favicon.ico
  82. @cherrypy.expose
  83. def favicon_ico(self):
  84. raise cherrypy.NotFound()
  85. # /version
  86. @cherrypy.expose
  87. @cherrypy.tools.json_out()
  88. def version(self):
  89. return self.server_version
  90. # /dbpath
  91. @cherrypy.expose
  92. @cherrypy.tools.json_out()
  93. def dbpath(self):
  94. return self.db.get_basepath()
  95. # /dbsize
  96. @cherrypy.expose
  97. @cherrypy.tools.json_out()
  98. def dbsize(self):
  99. return nilmdb.utils.du(self.db.get_basepath())
  100. class Stream(NilmApp):
  101. """Stream-specific operations"""
  102. # /stream/list
  103. # /stream/list?layout=PrepData
  104. # /stream/list?path=/newton/prep
  105. @cherrypy.expose
  106. @cherrypy.tools.json_out()
  107. def list(self, path = None, layout = None):
  108. """List all streams in the database. With optional path or
  109. layout parameter, just list streams that match the given path
  110. or layout"""
  111. return self.db.stream_list(path, layout)
  112. # /stream/create?path=/newton/prep&layout=PrepData
  113. @cherrypy.expose
  114. @cherrypy.tools.json_out()
  115. @exception_to_httperror(NilmDBError, ValueError)
  116. def create(self, path, layout):
  117. """Create a new stream in the database. Provide path
  118. and one of the nilmdb.layout.layouts keys.
  119. """
  120. return self.db.stream_create(path, layout)
  121. # /stream/destroy?path=/newton/prep
  122. @cherrypy.expose
  123. @cherrypy.tools.json_out()
  124. @exception_to_httperror(NilmDBError)
  125. def destroy(self, path):
  126. """Delete a stream and its associated data."""
  127. return self.db.stream_destroy(path)
  128. # /stream/get_metadata?path=/newton/prep
  129. # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
  130. @cherrypy.expose
  131. @cherrypy.tools.json_out()
  132. def get_metadata(self, path, key=None):
  133. """Get metadata for the named stream. If optional
  134. key parameters are specified, only return metadata
  135. matching the given keys."""
  136. try:
  137. data = self.db.stream_get_metadata(path)
  138. except nilmdb.server.nilmdb.StreamError as e:
  139. raise cherrypy.HTTPError("404 Not Found", e.message)
  140. if key is None: # If no keys specified, return them all
  141. key = data.keys()
  142. elif not isinstance(key, list):
  143. key = [ key ]
  144. result = {}
  145. for k in key:
  146. if k in data:
  147. result[k] = data[k]
  148. else: # Return "None" for keys with no matching value
  149. result[k] = None
  150. return result
  151. # /stream/set_metadata?path=/newton/prep&data=<json>
  152. @cherrypy.expose
  153. @cherrypy.tools.json_out()
  154. @exception_to_httperror(NilmDBError, LookupError, TypeError)
  155. def set_metadata(self, path, data):
  156. """Set metadata for the named stream, replacing any
  157. existing metadata. Data should be a json-encoded
  158. dictionary"""
  159. data_dict = json.loads(data)
  160. self.db.stream_set_metadata(path, data_dict)
  161. return "ok"
  162. # /stream/update_metadata?path=/newton/prep&data=<json>
  163. @cherrypy.expose
  164. @cherrypy.tools.json_out()
  165. @exception_to_httperror(NilmDBError, LookupError, TypeError)
  166. def update_metadata(self, path, data):
  167. """Update metadata for the named stream. Data
  168. should be a json-encoded dictionary"""
  169. data_dict = json.loads(data)
  170. self.db.stream_update_metadata(path, data_dict)
  171. return "ok"
  172. # /stream/insert?path=/newton/prep
  173. @cherrypy.expose
  174. @cherrypy.tools.json_out()
  175. #@cherrypy.tools.disable_prb()
  176. def insert(self, path, start, end):
  177. """
  178. Insert new data into the database. Provide textual data
  179. (matching the path's layout) as a HTTP PUT.
  180. """
  181. # Important that we always read the input before throwing any
  182. # errors, to keep lengths happy for persistent connections.
  183. # However, CherryPy 3.2.2 has a bug where this fails for GET
  184. # requests, so catch that. (issue #1134)
  185. try:
  186. body = cherrypy.request.body.read()
  187. except TypeError:
  188. raise cherrypy.HTTPError("400 Bad Request", "No request body")
  189. # Check path and get layout
  190. streams = self.db.stream_list(path = path)
  191. if len(streams) != 1:
  192. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  193. layout = streams[0][1]
  194. # Parse the input data
  195. try:
  196. parser = nilmdb.server.layout.Parser(layout)
  197. parser.parse(body)
  198. except nilmdb.server.layout.ParserError as e:
  199. raise cherrypy.HTTPError("400 Bad Request",
  200. "error parsing input data: " +
  201. e.message)
  202. if (not parser.min_timestamp or not parser.max_timestamp or
  203. not len(parser.data)):
  204. raise cherrypy.HTTPError("400 Bad Request",
  205. "no data provided")
  206. # Check limits
  207. start = float(start)
  208. end = float(end)
  209. if parser.min_timestamp < start:
  210. raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
  211. repr(parser.min_timestamp) +
  212. " < start time " + repr(start))
  213. if parser.max_timestamp >= end:
  214. raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
  215. repr(parser.max_timestamp) +
  216. " >= end time " + repr(end))
  217. # Now do the nilmdb insert, passing it the parser full of data.
  218. try:
  219. result = self.db.stream_insert(path, start, end, parser.data)
  220. except NilmDBError as e:
  221. raise cherrypy.HTTPError("400 Bad Request", e.message)
  222. # Done
  223. return "ok"
  224. # /stream/remove?path=/newton/prep
  225. # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
  226. @cherrypy.expose
  227. @cherrypy.tools.json_out()
  228. @exception_to_httperror(NilmDBError)
  229. def remove(self, path, start = None, end = None):
  230. """
  231. Remove data from the backend database. Removes all data in
  232. the interval [start, end). Returns the number of data points
  233. removed.
  234. """
  235. if start is not None:
  236. start = float(start)
  237. if end is not None:
  238. end = float(end)
  239. if start is not None and end is not None:
  240. if end < start:
  241. raise cherrypy.HTTPError("400 Bad Request",
  242. "end before start")
  243. return self.db.stream_remove(path, start, end)
  244. # /stream/intervals?path=/newton/prep
  245. # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
  246. @cherrypy.expose
  247. @chunked_response
  248. @response_type("text/plain")
  249. def intervals(self, path, start = None, end = None):
  250. """
  251. Get intervals from backend database. Streams the resulting
  252. intervals as JSON strings separated by newlines. This may
  253. make multiple requests to the nilmdb backend to avoid causing
  254. it to block for too long.
  255. Note that the response type is set to 'text/plain' even
  256. though we're sending back JSON; this is because we're not
  257. really returning a single JSON object.
  258. """
  259. if start is not None:
  260. start = float(start)
  261. if end is not None:
  262. end = float(end)
  263. if start is not None and end is not None:
  264. if end < start:
  265. raise cherrypy.HTTPError("400 Bad Request",
  266. "end before start")
  267. streams = self.db.stream_list(path = path)
  268. if len(streams) != 1:
  269. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  270. @workaround_cp_bug_1200
  271. def content(start, end):
  272. # Note: disable chunked responses to see tracebacks from here.
  273. while True:
  274. (intervals, restart) = self.db.stream_intervals(path,start,end)
  275. response = ''.join([ json.dumps(i) + "\n" for i in intervals ])
  276. yield response
  277. if restart == 0:
  278. break
  279. start = restart
  280. return content(start, end)
  281. # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
  282. @cherrypy.expose
  283. @chunked_response
  284. @response_type("text/plain")
  285. def extract(self, path, start = None, end = None, count = False):
  286. """
  287. Extract data from backend database. Streams the resulting
  288. entries as ASCII text lines separated by newlines. This may
  289. make multiple requests to the nilmdb backend to avoid causing
  290. it to block for too long.
  291. Add count=True to return a count rather than actual data.
  292. """
  293. if start is not None:
  294. start = float(start)
  295. if end is not None:
  296. end = float(end)
  297. # Check parameters
  298. if start is not None and end is not None:
  299. if end < start:
  300. raise cherrypy.HTTPError("400 Bad Request",
  301. "end before start")
  302. # Check path and get layout
  303. streams = self.db.stream_list(path = path)
  304. if len(streams) != 1:
  305. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  306. layout = streams[0][1]
  307. # Get formatter
  308. formatter = nilmdb.server.layout.Formatter(layout)
  309. @workaround_cp_bug_1200
  310. def content(start, end, count):
  311. # Note: disable chunked responses to see tracebacks from here.
  312. if count:
  313. matched = self.db.stream_extract(path, start, end, count)
  314. yield sprintf("%d\n", matched)
  315. return
  316. while True:
  317. (data, restart) = self.db.stream_extract(path, start, end)
  318. # Format the data and yield it
  319. yield formatter.format(data)
  320. if restart == 0:
  321. return
  322. start = restart
  323. return content(start, end, count)
  324. class Exiter(object):
  325. """App that exits the server, for testing"""
  326. @cherrypy.expose
  327. def index(self):
  328. cherrypy.response.headers['Content-Type'] = 'text/plain'
  329. def content():
  330. yield 'Exiting by request'
  331. raise SystemExit
  332. return content()
  333. index._cp_config = { 'response.stream': True }
  334. class Server(object):
  335. def __init__(self, db, host = '127.0.0.1', port = 8080,
  336. stoppable = False, # whether /exit URL exists
  337. embedded = True, # hide diagnostics and output, etc
  338. fast_shutdown = False, # don't wait for clients to disconn.
  339. force_traceback = False # include traceback in all errors
  340. ):
  341. self.version = version
  342. # Need to wrap DB object in a serializer because we'll call
  343. # into it from separate threads.
  344. self.embedded = embedded
  345. self.db = nilmdb.utils.Serializer(db)
  346. cherrypy.config.update({
  347. 'server.socket_host': host,
  348. 'server.socket_port': port,
  349. 'engine.autoreload_on': False,
  350. 'server.max_request_body_size': 4*1024*1024,
  351. 'error_page.default': self.json_error_page,
  352. })
  353. if self.embedded:
  354. cherrypy.config.update({ 'environment': 'embedded' })
  355. # Send a permissive Access-Control-Allow-Origin (CORS) header
  356. # with all responses so that browsers can send cross-domain
  357. # requests to this server.
  358. cherrypy.config.update({ 'response.headers.Access-Control-Allow-Origin':
  359. '*' })
  360. # Send tracebacks in error responses. They're hidden by the
  361. # error_page function for client errors (code 400-499).
  362. cherrypy.config.update({ 'request.show_tracebacks' : True })
  363. self.force_traceback = force_traceback
  364. # Patch CherryPy error handler to never pad out error messages.
  365. # This isn't necessary, but then again, neither is padding the
  366. # error messages.
  367. cherrypy._cperror._ie_friendly_error_sizes = {}
  368. cherrypy.tree.apps = {}
  369. cherrypy.tree.mount(Root(self.db, self.version), "/")
  370. cherrypy.tree.mount(Stream(self.db), "/stream")
  371. if stoppable:
  372. cherrypy.tree.mount(Exiter(), "/exit")
  373. # Shutdowns normally wait for clients to disconnect. To speed
  374. # up tests, set fast_shutdown = True
  375. if fast_shutdown:
  376. # Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
  377. cherrypy.server.shutdown_timeout = 0.01
  378. else:
  379. cherrypy.server.shutdown_timeout = 5
  380. def json_error_page(self, status, message, traceback, version):
  381. """Return a custom error page in JSON so the client can parse it"""
  382. errordata = { "status" : status,
  383. "message" : message,
  384. "traceback" : traceback }
  385. # Don't send a traceback if the error was 400-499 (client's fault)
  386. try:
  387. code = int(status.split()[0])
  388. if not self.force_traceback:
  389. if code >= 400 and code <= 499:
  390. errordata["traceback"] = ""
  391. except Exception as e: # pragma: no cover
  392. pass
  393. # Override the response type, which was previously set to text/html
  394. cherrypy.serving.response.headers['Content-Type'] = (
  395. "application/json;charset=utf-8" )
  396. # Undo the HTML escaping that cherrypy's get_error_page function applies
  397. # (cherrypy issue 1135)
  398. for k, v in errordata.iteritems():
  399. v = v.replace("&lt;","<")
  400. v = v.replace("&gt;",">")
  401. v = v.replace("&amp;","&")
  402. errordata[k] = v
  403. return json.dumps(errordata, separators=(',',':'))
  404. def start(self, blocking = False, event = None):
  405. if not self.embedded: # pragma: no cover
  406. # Handle signals nicely
  407. if hasattr(cherrypy.engine, "signal_handler"):
  408. cherrypy.engine.signal_handler.subscribe()
  409. if hasattr(cherrypy.engine, "console_control_handler"):
  410. cherrypy.engine.console_control_handler.subscribe()
  411. # Cherrypy stupidly calls os._exit(70) when it can't bind the
  412. # port. At least try to print a reasonable error and continue
  413. # in this case, rather than just dying silently (as we would
  414. # otherwise do in embedded mode)
  415. real_exit = os._exit
  416. def fake_exit(code): # pragma: no cover
  417. if code == os.EX_SOFTWARE:
  418. fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
  419. else:
  420. real_exit(code)
  421. os._exit = fake_exit
  422. cherrypy.engine.start()
  423. os._exit = real_exit
  424. # Signal that the engine has started successfully
  425. if event is not None:
  426. event.set()
  427. if blocking:
  428. try:
  429. cherrypy.engine.wait(cherrypy.engine.states.EXITING,
  430. interval = 0.1, channel = 'main')
  431. except (KeyboardInterrupt, IOError): # pragma: no cover
  432. cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
  433. cherrypy.engine.exit()
  434. except SystemExit: # pragma: no cover
  435. cherrypy.engine.log('SystemExit raised: shutting down bus')
  436. cherrypy.engine.exit()
  437. raise
  438. def stop(self):
  439. cherrypy.engine.exit()