You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

572 lines
21 KiB

  1. """CherryPy-based server for accessing NILM database via HTTP"""
  2. # Need absolute_import so that "import nilmdb" won't pull in
  3. # nilmdb.py, but will pull the nilmdb module instead.
  4. from __future__ import absolute_import
  5. import nilmdb.server
  6. from nilmdb.utils.printf import *
  7. from nilmdb.server.errors import NilmDBError
  8. import cherrypy
  9. import sys
  10. import os
  11. import simplejson as json
  12. import decorator
  13. import psutil
  14. class NilmApp(object):
  15. def __init__(self, db):
  16. self.db = db
  17. # Decorators
  18. def chunked_response(func):
  19. """Decorator to enable chunked responses."""
  20. # Set this to False to get better tracebacks from some requests
  21. # (/stream/extract, /stream/intervals).
  22. func._cp_config = { 'response.stream': True }
  23. return func
  24. def response_type(content_type):
  25. """Return a decorator-generating function that sets the
  26. response type to the specified string."""
  27. def wrapper(func, *args, **kwargs):
  28. cherrypy.response.headers['Content-Type'] = content_type
  29. return func(*args, **kwargs)
  30. return decorator.decorator(wrapper)
  31. @decorator.decorator
  32. def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
  33. """Decorator to work around CherryPy bug #1200 in a response
  34. generator.
  35. Even if chunked responses are disabled, LookupError or
  36. UnicodeError exceptions may still be swallowed by CherryPy due to
  37. bug #1200. This throws them as generic Exceptions instead so that
  38. they make it through.
  39. """
  40. exc_info = None
  41. try:
  42. for val in func(*args, **kwargs):
  43. yield val
  44. except (LookupError, UnicodeError):
  45. # Re-raise it, but maintain the original traceback
  46. exc_info = sys.exc_info()
  47. new_exc = Exception(exc_info[0].__name__ + ": " + str(exc_info[1]))
  48. raise new_exc, None, exc_info[2]
  49. finally:
  50. del exc_info
  51. def exception_to_httperror(*expected):
  52. """Return a decorator-generating function that catches expected
  53. errors and throws a HTTPError describing it instead.
  54. @exception_to_httperror(NilmDBError, ValueError)
  55. def foo():
  56. pass
  57. """
  58. def wrapper(func, *args, **kwargs):
  59. exc_info = None
  60. try:
  61. return func(*args, **kwargs)
  62. except expected:
  63. # Re-raise it, but maintain the original traceback
  64. exc_info = sys.exc_info()
  65. new_exc = cherrypy.HTTPError("400 Bad Request", str(exc_info[1]))
  66. raise new_exc, None, exc_info[2]
  67. finally:
  68. del exc_info
  69. # We need to preserve the function's argspecs for CherryPy to
  70. # handle argument errors correctly. Decorator.decorator takes
  71. # care of that.
  72. return decorator.decorator(wrapper)
  73. # Custom CherryPy tools
  74. def CORS_allow(methods):
  75. """This does several things:
  76. Handles CORS preflight requests.
  77. Adds Allow: header to all requests.
  78. Raise 405 if request.method not in method.
  79. It is similar to cherrypy.tools.allow, with the CORS stuff added.
  80. """
  81. request = cherrypy.request.headers
  82. response = cherrypy.response.headers
  83. if not isinstance(methods, (tuple, list)): # pragma: no cover
  84. methods = [ methods ]
  85. methods = [ m.upper() for m in methods if m ]
  86. if not methods: # pragma: no cover
  87. methods = [ 'GET', 'HEAD' ]
  88. elif 'GET' in methods and 'HEAD' not in methods: # pragma: no cover
  89. methods.append('HEAD')
  90. response['Allow'] = ', '.join(methods)
  91. # Allow all origins
  92. if 'Origin' in request:
  93. response['Access-Control-Allow-Origin'] = request['Origin']
  94. # If it's a CORS request, send response.
  95. request_method = request.get("Access-Control-Request-Method", None)
  96. request_headers = request.get("Access-Control-Request-Headers", None)
  97. if (cherrypy.request.method == "OPTIONS" and
  98. request_method and request_headers):
  99. response['Access-Control-Allow-Headers'] = request_headers
  100. response['Access-Control-Allow-Methods'] = ', '.join(methods)
  101. # Try to stop further processing and return a 200 OK
  102. cherrypy.response.status = "200 OK"
  103. cherrypy.response.body = ""
  104. cherrypy.request.handler = lambda: ""
  105. return
  106. # Reject methods that were not explicitly allowed
  107. if cherrypy.request.method not in methods:
  108. raise cherrypy.HTTPError(405)
  109. cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
  110. # Helper for json_in tool to process JSON data into normal request
  111. # parameters.
  112. def json_to_request_params(body):
  113. cherrypy.lib.jsontools.json_processor(body)
  114. if not isinstance(cherrypy.request.json, dict):
  115. raise cherrypy.HTTPError(415)
  116. cherrypy.request.params.update(cherrypy.request.json)
  117. # CherryPy apps
  118. class Root(NilmApp):
  119. """Root application for NILM database"""
  120. def __init__(self, db):
  121. super(Root, self).__init__(db)
  122. # /
  123. @cherrypy.expose
  124. def index(self):
  125. raise cherrypy.NotFound()
  126. # /favicon.ico
  127. @cherrypy.expose
  128. def favicon_ico(self):
  129. raise cherrypy.NotFound()
  130. # /version
  131. @cherrypy.expose
  132. @cherrypy.tools.json_out()
  133. def version(self):
  134. return nilmdb.__version__
  135. # /dbinfo
  136. @cherrypy.expose
  137. @cherrypy.tools.json_out()
  138. def dbinfo(self):
  139. """Return a dictionary with the database path,
  140. size of the database in bytes, and free disk space in bytes"""
  141. path = self.db.get_basepath()
  142. return { "path": path,
  143. "size": nilmdb.utils.du(path),
  144. "free": psutil.disk_usage(path).free }
  145. class Stream(NilmApp):
  146. """Stream-specific operations"""
  147. # /stream/list
  148. # /stream/list?layout=float32_8
  149. # /stream/list?path=/newton/prep&extent=1
  150. @cherrypy.expose
  151. @cherrypy.tools.json_out()
  152. def list(self, path = None, layout = None, extent = None):
  153. """List all streams in the database. With optional path or
  154. layout parameter, just list streams that match the given path
  155. or layout.
  156. If extent is not given, returns a list of lists containing
  157. the path and layout: [ path, layout ]
  158. If extent is provided, returns a list of lists containing the
  159. path, layout, and min/max extent of the data:
  160. [ path, layout, extent_min, extent_max ]
  161. """
  162. return self.db.stream_list(path, layout, bool(extent))
  163. # /stream/create?path=/newton/prep&layout=float32_8
  164. @cherrypy.expose
  165. @cherrypy.tools.json_in()
  166. @cherrypy.tools.json_out()
  167. @exception_to_httperror(NilmDBError, ValueError)
  168. @cherrypy.tools.CORS_allow(methods = ["POST"])
  169. def create(self, path, layout):
  170. """Create a new stream in the database. Provide path
  171. and one of the nilmdb.layout.layouts keys.
  172. """
  173. return self.db.stream_create(path, layout)
  174. # /stream/destroy?path=/newton/prep
  175. @cherrypy.expose
  176. @cherrypy.tools.json_in()
  177. @cherrypy.tools.json_out()
  178. @exception_to_httperror(NilmDBError)
  179. @cherrypy.tools.CORS_allow(methods = ["POST"])
  180. def destroy(self, path):
  181. """Delete a stream and its associated data."""
  182. return self.db.stream_destroy(path)
  183. # /stream/get_metadata?path=/newton/prep
  184. # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
  185. @cherrypy.expose
  186. @cherrypy.tools.json_out()
  187. def get_metadata(self, path, key=None):
  188. """Get metadata for the named stream. If optional
  189. key parameters are specified, only return metadata
  190. matching the given keys."""
  191. try:
  192. data = self.db.stream_get_metadata(path)
  193. except nilmdb.server.nilmdb.StreamError as e:
  194. raise cherrypy.HTTPError("404 Not Found", e.message)
  195. if key is None: # If no keys specified, return them all
  196. key = data.keys()
  197. elif not isinstance(key, list):
  198. key = [ key ]
  199. result = {}
  200. for k in key:
  201. if k in data:
  202. result[k] = data[k]
  203. else: # Return "None" for keys with no matching value
  204. result[k] = None
  205. return result
  206. # Helper for set_metadata and get_metadata
  207. def _metadata_helper(self, function, path, data):
  208. if not isinstance(data, dict):
  209. try:
  210. data = dict(json.loads(data))
  211. except TypeError as e:
  212. raise NilmDBError("can't parse 'data' parameter: " + e.message)
  213. for key in data:
  214. if not (isinstance(data[key], basestring) or
  215. isinstance(data[key], float) or
  216. isinstance(data[key], int)):
  217. raise NilmDBError("metadata values must be a string or number")
  218. function(path, data)
  219. # /stream/set_metadata?path=/newton/prep&data=<json>
  220. @cherrypy.expose
  221. @cherrypy.tools.json_in()
  222. @cherrypy.tools.json_out()
  223. @exception_to_httperror(NilmDBError, LookupError)
  224. @cherrypy.tools.CORS_allow(methods = ["POST"])
  225. def set_metadata(self, path, data):
  226. """Set metadata for the named stream, replacing any existing
  227. metadata. Data can be json-encoded or a plain dictionary."""
  228. self._metadata_helper(self.db.stream_set_metadata, path, data)
  229. # /stream/update_metadata?path=/newton/prep&data=<json>
  230. @cherrypy.expose
  231. @cherrypy.tools.json_in()
  232. @cherrypy.tools.json_out()
  233. @exception_to_httperror(NilmDBError, LookupError, ValueError)
  234. @cherrypy.tools.CORS_allow(methods = ["POST"])
  235. def update_metadata(self, path, data):
  236. """Set metadata for the named stream, replacing any existing
  237. metadata. Data can be json-encoded or a plain dictionary."""
  238. self._metadata_helper(self.db.stream_update_metadata, path, data)
  239. # /stream/insert?path=/newton/prep
  240. @cherrypy.expose
  241. @cherrypy.tools.json_out()
  242. @exception_to_httperror(NilmDBError, ValueError)
  243. @cherrypy.tools.CORS_allow(methods = ["PUT"])
  244. def insert(self, path, start, end):
  245. """
  246. Insert new data into the database. Provide textual data
  247. (matching the path's layout) as a HTTP PUT.
  248. """
  249. # Important that we always read the input before throwing any
  250. # errors, to keep lengths happy for persistent connections.
  251. # Note that CherryPy 3.2.2 has a bug where this fails for GET
  252. # requests, if we ever want to handle those (issue #1134)
  253. body = cherrypy.request.body.read()
  254. # Check path and get layout
  255. streams = self.db.stream_list(path = path)
  256. if len(streams) != 1:
  257. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  258. # Check limits
  259. start = float(start)
  260. end = float(end)
  261. if start >= end:
  262. raise cherrypy.HTTPError("400 Bad Request",
  263. "start must precede end")
  264. # Pass the data directly to nilmdb, which will parse it and
  265. # raise a ValueError if there are any problems.
  266. self.db.stream_insert(path, start, end, body)
  267. # Done
  268. return
  269. # /stream/remove?path=/newton/prep
  270. # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
  271. @cherrypy.expose
  272. @cherrypy.tools.json_in()
  273. @cherrypy.tools.json_out()
  274. @exception_to_httperror(NilmDBError)
  275. @cherrypy.tools.CORS_allow(methods = ["POST"])
  276. def remove(self, path, start = None, end = None):
  277. """
  278. Remove data from the backend database. Removes all data in
  279. the interval [start, end). Returns the number of data points
  280. removed.
  281. """
  282. if start is not None:
  283. start = float(start)
  284. if end is not None:
  285. end = float(end)
  286. if start is not None and end is not None:
  287. if start >= end:
  288. raise cherrypy.HTTPError("400 Bad Request",
  289. "start must precede end")
  290. return self.db.stream_remove(path, start, end)
  291. # /stream/intervals?path=/newton/prep
  292. # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
  293. @cherrypy.expose
  294. @chunked_response
  295. @response_type("application/x-json-stream")
  296. def intervals(self, path, start = None, end = None):
  297. """
  298. Get intervals from backend database. Streams the resulting
  299. intervals as JSON strings separated by CR LF pairs. This may
  300. make multiple requests to the nilmdb backend to avoid causing
  301. it to block for too long.
  302. Note that the response type is the non-standard
  303. 'application/x-json-stream' for lack of a better option.
  304. """
  305. if start is not None:
  306. start = float(start)
  307. if end is not None:
  308. end = float(end)
  309. if start is not None and end is not None:
  310. if start >= end:
  311. raise cherrypy.HTTPError("400 Bad Request",
  312. "start must precede end")
  313. streams = self.db.stream_list(path = path)
  314. if len(streams) != 1:
  315. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  316. @workaround_cp_bug_1200
  317. def content(start, end):
  318. # Note: disable chunked responses to see tracebacks from here.
  319. while True:
  320. (ints, restart) = self.db.stream_intervals(path, start, end)
  321. response = ''.join([ json.dumps(i) + "\r\n" for i in ints ])
  322. yield response
  323. if restart == 0:
  324. break
  325. start = restart
  326. return content(start, end)
  327. # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
  328. @cherrypy.expose
  329. @chunked_response
  330. @response_type("text/plain")
  331. def extract(self, path, start = None, end = None, count = False):
  332. """
  333. Extract data from backend database. Streams the resulting
  334. entries as ASCII text lines separated by newlines. This may
  335. make multiple requests to the nilmdb backend to avoid causing
  336. it to block for too long.
  337. Add count=True to return a count rather than actual data.
  338. """
  339. if start is not None:
  340. start = float(start)
  341. if end is not None:
  342. end = float(end)
  343. # Check parameters
  344. if start is not None and end is not None:
  345. if start >= end:
  346. raise cherrypy.HTTPError("400 Bad Request",
  347. "start must precede end")
  348. # Check path and get layout
  349. streams = self.db.stream_list(path = path)
  350. if len(streams) != 1:
  351. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  352. @workaround_cp_bug_1200
  353. def content(start, end, count):
  354. # Note: disable chunked responses to see tracebacks from here.
  355. if count:
  356. matched = self.db.stream_extract(path, start, end, count)
  357. yield sprintf("%d\n", matched)
  358. return
  359. while True:
  360. (data, restart) = self.db.stream_extract(path, start, end)
  361. yield data
  362. if restart == 0:
  363. return
  364. start = restart
  365. return content(start, end, count)
  366. class Exiter(object):
  367. """App that exits the server, for testing"""
  368. @cherrypy.expose
  369. def index(self):
  370. cherrypy.response.headers['Content-Type'] = 'text/plain'
  371. def content():
  372. yield 'Exiting by request'
  373. raise SystemExit
  374. return content()
  375. index._cp_config = { 'response.stream': True }
  376. class Server(object):
  377. def __init__(self, db, host = '127.0.0.1', port = 8080,
  378. stoppable = False, # whether /exit URL exists
  379. embedded = True, # hide diagnostics and output, etc
  380. fast_shutdown = False, # don't wait for clients to disconn.
  381. force_traceback = False # include traceback in all errors
  382. ):
  383. # Save server version, just for verification during tests
  384. self.version = nilmdb.__version__
  385. self.embedded = embedded
  386. self.db = db
  387. if not getattr(db, "_thread_safe", None):
  388. raise KeyError("Database object " + str(db) + " doesn't claim "
  389. "to be thread safe. You should pass "
  390. "nilmdb.utils.serializer_proxy(NilmDB)(args) "
  391. "rather than NilmDB(args).")
  392. # Build up global server configuration
  393. cherrypy.config.update({
  394. 'server.socket_host': host,
  395. 'server.socket_port': port,
  396. 'engine.autoreload_on': False,
  397. 'server.max_request_body_size': 8*1024*1024,
  398. })
  399. if self.embedded:
  400. cherrypy.config.update({ 'environment': 'embedded' })
  401. # Build up application specific configuration
  402. app_config = {}
  403. app_config.update({
  404. 'error_page.default': self.json_error_page,
  405. })
  406. # Some default headers to just help identify that things are working
  407. app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
  408. # Set up Cross-Origin Resource Sharing (CORS) handler so we
  409. # can correctly respond to browsers' CORS preflight requests.
  410. # This also limits verbs to GET and HEAD by default.
  411. app_config.update({ 'tools.CORS_allow.on': True,
  412. 'tools.CORS_allow.methods': ['GET', 'HEAD'] })
  413. # Configure the 'json_in' tool to also allow other content-types
  414. # (like x-www-form-urlencoded), and to treat JSON as a dict that
  415. # fills requests.param.
  416. app_config.update({ 'tools.json_in.force': False,
  417. 'tools.json_in.processor': json_to_request_params })
  418. # Send tracebacks in error responses. They're hidden by the
  419. # error_page function for client errors (code 400-499).
  420. app_config.update({ 'request.show_tracebacks' : True })
  421. self.force_traceback = force_traceback
  422. # Patch CherryPy error handler to never pad out error messages.
  423. # This isn't necessary, but then again, neither is padding the
  424. # error messages.
  425. cherrypy._cperror._ie_friendly_error_sizes = {}
  426. # Build up the application and mount it
  427. root = Root(self.db)
  428. root.stream = Stream(self.db)
  429. if stoppable:
  430. root.exit = Exiter()
  431. cherrypy.tree.apps = {}
  432. cherrypy.tree.mount(root, "/", config = { "/" : app_config })
  433. # Shutdowns normally wait for clients to disconnect. To speed
  434. # up tests, set fast_shutdown = True
  435. if fast_shutdown:
  436. # Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
  437. cherrypy.server.shutdown_timeout = 0.01
  438. else:
  439. cherrypy.server.shutdown_timeout = 5
  440. def json_error_page(self, status, message, traceback, version):
  441. """Return a custom error page in JSON so the client can parse it"""
  442. errordata = { "status" : status,
  443. "message" : message,
  444. "traceback" : traceback }
  445. # Don't send a traceback if the error was 400-499 (client's fault)
  446. try:
  447. code = int(status.split()[0])
  448. if not self.force_traceback:
  449. if code >= 400 and code <= 499:
  450. errordata["traceback"] = ""
  451. except Exception: # pragma: no cover
  452. pass
  453. # Override the response type, which was previously set to text/html
  454. cherrypy.serving.response.headers['Content-Type'] = (
  455. "application/json;charset=utf-8" )
  456. # Undo the HTML escaping that cherrypy's get_error_page function applies
  457. # (cherrypy issue 1135)
  458. for k, v in errordata.iteritems():
  459. v = v.replace("&lt;","<")
  460. v = v.replace("&gt;",">")
  461. v = v.replace("&amp;","&")
  462. errordata[k] = v
  463. return json.dumps(errordata, separators=(',',':'))
  464. def start(self, blocking = False, event = None):
  465. if not self.embedded: # pragma: no cover
  466. # Handle signals nicely
  467. if hasattr(cherrypy.engine, "signal_handler"):
  468. cherrypy.engine.signal_handler.subscribe()
  469. if hasattr(cherrypy.engine, "console_control_handler"):
  470. cherrypy.engine.console_control_handler.subscribe()
  471. # Cherrypy stupidly calls os._exit(70) when it can't bind the
  472. # port. At least try to print a reasonable error and continue
  473. # in this case, rather than just dying silently (as we would
  474. # otherwise do in embedded mode)
  475. real_exit = os._exit
  476. def fake_exit(code): # pragma: no cover
  477. if code == os.EX_SOFTWARE:
  478. fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
  479. else:
  480. real_exit(code)
  481. os._exit = fake_exit
  482. cherrypy.engine.start()
  483. os._exit = real_exit
  484. # Signal that the engine has started successfully
  485. if event is not None:
  486. event.set()
  487. if blocking:
  488. try:
  489. cherrypy.engine.wait(cherrypy.engine.states.EXITING,
  490. interval = 0.1, channel = 'main')
  491. except (KeyboardInterrupt, IOError): # pragma: no cover
  492. cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
  493. cherrypy.engine.exit()
  494. except SystemExit: # pragma: no cover
  495. cherrypy.engine.log('SystemExit raised: shutting down bus')
  496. cherrypy.engine.exit()
  497. raise
  498. def stop(self):
  499. cherrypy.engine.exit()