You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

514 lines
19 KiB

  1. """CherryPy-based server for accessing NILM database via HTTP"""
  2. # Need absolute_import so that "import nilmdb" won't pull in
  3. # nilmdb.py, but will pull the nilmdb module instead.
  4. from __future__ import absolute_import
  5. import nilmdb
  6. from nilmdb.utils.printf import *
  7. from nilmdb.server.errors import NilmDBError
  8. import cherrypy
  9. import sys
  10. import os
  11. import simplejson as json
  12. import decorator
  13. import traceback
  14. import psutil
  15. class NilmApp(object):
  16. def __init__(self, db):
  17. self.db = db
  18. # Decorators
  19. def chunked_response(func):
  20. """Decorator to enable chunked responses."""
  21. # Set this to False to get better tracebacks from some requests
  22. # (/stream/extract, /stream/intervals).
  23. func._cp_config = { 'response.stream': True }
  24. return func
  25. def response_type(content_type):
  26. """Return a decorator-generating function that sets the
  27. response type to the specified string."""
  28. def wrapper(func, *args, **kwargs):
  29. cherrypy.response.headers['Content-Type'] = content_type
  30. return func(*args, **kwargs)
  31. return decorator.decorator(wrapper)
  32. @decorator.decorator
  33. def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
  34. """Decorator to work around CherryPy bug #1200 in a response
  35. generator.
  36. Even if chunked responses are disabled, LookupError or
  37. UnicodeError exceptions may still be swallowed by CherryPy due to
  38. bug #1200. This throws them as generic Exceptions instead so that
  39. they make it through.
  40. """
  41. try:
  42. for val in func(*args, **kwargs):
  43. yield val
  44. except (LookupError, UnicodeError):
  45. raise Exception("bug workaround; real exception is:\n" +
  46. traceback.format_exc())
  47. def exception_to_httperror(*expected):
  48. """Return a decorator-generating function that catches expected
  49. errors and throws a HTTPError describing it instead.
  50. @exception_to_httperror(NilmDBError, ValueError)
  51. def foo():
  52. pass
  53. """
  54. def wrapper(func, *args, **kwargs):
  55. try:
  56. return func(*args, **kwargs)
  57. except expected as e:
  58. message = sprintf("%s", str(e))
  59. raise cherrypy.HTTPError("400 Bad Request", message)
  60. # We need to preserve the function's argspecs for CherryPy to
  61. # handle argument errors correctly. Decorator.decorator takes
  62. # care of that.
  63. return decorator.decorator(wrapper)
  64. # CherryPy apps
  65. class Root(NilmApp):
  66. """Root application for NILM database"""
  67. def __init__(self, db):
  68. super(Root, self).__init__(db)
  69. # /
  70. @cherrypy.expose
  71. def index(self):
  72. raise cherrypy.NotFound()
  73. # /favicon.ico
  74. @cherrypy.expose
  75. def favicon_ico(self):
  76. raise cherrypy.NotFound()
  77. # /version
  78. @cherrypy.expose
  79. @cherrypy.tools.json_out()
  80. def version(self):
  81. return nilmdb.__version__
  82. # /dbinfo
  83. @cherrypy.expose
  84. @cherrypy.tools.json_out()
  85. def dbinfo(self):
  86. """Return a dictionary with the database path,
  87. size of the database in bytes, and free disk space in bytes"""
  88. path = self.db.get_basepath()
  89. return { "path": path,
  90. "size": nilmdb.utils.du(path),
  91. "free": psutil.disk_usage(path).free }
  92. class Stream(NilmApp):
  93. """Stream-specific operations"""
  94. # /stream/list
  95. # /stream/list?layout=PrepData
  96. # /stream/list?path=/newton/prep&extent=1
  97. @cherrypy.expose
  98. @cherrypy.tools.json_out()
  99. def list(self, path = None, layout = None, extent = None):
  100. """List all streams in the database. With optional path or
  101. layout parameter, just list streams that match the given path
  102. or layout.
  103. If extent is not given, returns a list of lists containing
  104. the path and layout: [ path, layout ]
  105. If extent is provided, returns a list of lists containing the
  106. path, layout, and min/max extent of the data:
  107. [ path, layout, extent_min, extent_max ]
  108. """
  109. return self.db.stream_list(path, layout, bool(extent))
  110. # /stream/create?path=/newton/prep&layout=PrepData
  111. @cherrypy.expose
  112. @cherrypy.tools.json_out()
  113. @exception_to_httperror(NilmDBError, ValueError)
  114. @cherrypy.tools.allow(methods = ["POST"])
  115. def create(self, path, layout):
  116. """Create a new stream in the database. Provide path
  117. and one of the nilmdb.layout.layouts keys.
  118. """
  119. return self.db.stream_create(path, layout)
  120. # /stream/destroy?path=/newton/prep
  121. @cherrypy.expose
  122. @cherrypy.tools.json_out()
  123. @exception_to_httperror(NilmDBError)
  124. @cherrypy.tools.allow(methods = ["POST"])
  125. def destroy(self, path):
  126. """Delete a stream and its associated data."""
  127. return self.db.stream_destroy(path)
  128. # /stream/get_metadata?path=/newton/prep
  129. # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
  130. @cherrypy.expose
  131. @cherrypy.tools.json_out()
  132. def get_metadata(self, path, key=None):
  133. """Get metadata for the named stream. If optional
  134. key parameters are specified, only return metadata
  135. matching the given keys."""
  136. try:
  137. data = self.db.stream_get_metadata(path)
  138. except nilmdb.server.nilmdb.StreamError as e:
  139. raise cherrypy.HTTPError("404 Not Found", e.message)
  140. if key is None: # If no keys specified, return them all
  141. key = data.keys()
  142. elif not isinstance(key, list):
  143. key = [ key ]
  144. result = {}
  145. for k in key:
  146. if k in data:
  147. result[k] = data[k]
  148. else: # Return "None" for keys with no matching value
  149. result[k] = None
  150. return result
  151. # /stream/set_metadata?path=/newton/prep&data=<json>
  152. @cherrypy.expose
  153. @cherrypy.tools.json_out()
  154. @exception_to_httperror(NilmDBError, LookupError, TypeError)
  155. @cherrypy.tools.allow(methods = ["POST"])
  156. def set_metadata(self, path, data):
  157. """Set metadata for the named stream, replacing any
  158. existing metadata. Data should be a json-encoded
  159. dictionary"""
  160. data_dict = json.loads(data)
  161. self.db.stream_set_metadata(path, data_dict)
  162. # /stream/update_metadata?path=/newton/prep&data=<json>
  163. @cherrypy.expose
  164. @cherrypy.tools.json_out()
  165. @exception_to_httperror(NilmDBError, LookupError, TypeError)
  166. @cherrypy.tools.allow(methods = ["POST"])
  167. def update_metadata(self, path, data):
  168. """Update metadata for the named stream. Data
  169. should be a json-encoded dictionary"""
  170. data_dict = json.loads(data)
  171. self.db.stream_update_metadata(path, data_dict)
  172. # /stream/insert?path=/newton/prep
  173. @cherrypy.expose
  174. @cherrypy.tools.json_out()
  175. @cherrypy.tools.allow(methods = ["PUT"])
  176. def insert(self, path, start, end):
  177. """
  178. Insert new data into the database. Provide textual data
  179. (matching the path's layout) as a HTTP PUT.
  180. """
  181. # Important that we always read the input before throwing any
  182. # errors, to keep lengths happy for persistent connections.
  183. # Note that CherryPy 3.2.2 has a bug where this fails for GET
  184. # requests, if we ever want to handle those (issue #1134)
  185. body = cherrypy.request.body.read()
  186. # Check path and get layout
  187. streams = self.db.stream_list(path = path)
  188. if len(streams) != 1:
  189. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  190. layout = streams[0][1]
  191. # Parse the input data
  192. try:
  193. parser = nilmdb.server.layout.Parser(layout)
  194. parser.parse(body)
  195. except nilmdb.server.layout.ParserError as e:
  196. raise cherrypy.HTTPError("400 Bad Request",
  197. "error parsing input data: " +
  198. e.message)
  199. # Check limits
  200. start = float(start)
  201. end = float(end)
  202. if start >= end:
  203. raise cherrypy.HTTPError("400 Bad Request",
  204. "start must precede end")
  205. if parser.min_timestamp is not None and parser.min_timestamp < start:
  206. raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
  207. repr(parser.min_timestamp) +
  208. " < start time " + repr(start))
  209. if parser.max_timestamp is not None and parser.max_timestamp >= end:
  210. raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
  211. repr(parser.max_timestamp) +
  212. " >= end time " + repr(end))
  213. # Now do the nilmdb insert, passing it the parser full of data.
  214. try:
  215. self.db.stream_insert(path, start, end, parser.data)
  216. except NilmDBError as e:
  217. raise cherrypy.HTTPError("400 Bad Request", e.message)
  218. # Done
  219. return
  220. # /stream/remove?path=/newton/prep
  221. # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
  222. @cherrypy.expose
  223. @cherrypy.tools.json_out()
  224. @exception_to_httperror(NilmDBError)
  225. @cherrypy.tools.allow(methods = ["POST"])
  226. def remove(self, path, start = None, end = None):
  227. """
  228. Remove data from the backend database. Removes all data in
  229. the interval [start, end). Returns the number of data points
  230. removed.
  231. """
  232. if start is not None:
  233. start = float(start)
  234. if end is not None:
  235. end = float(end)
  236. if start is not None and end is not None:
  237. if start >= end:
  238. raise cherrypy.HTTPError("400 Bad Request",
  239. "start must precede end")
  240. return self.db.stream_remove(path, start, end)
  241. # /stream/intervals?path=/newton/prep
  242. # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
  243. @cherrypy.expose
  244. @chunked_response
  245. @response_type("application/x-json-stream")
  246. def intervals(self, path, start = None, end = None):
  247. """
  248. Get intervals from backend database. Streams the resulting
  249. intervals as JSON strings separated by CR LF pairs. This may
  250. make multiple requests to the nilmdb backend to avoid causing
  251. it to block for too long.
  252. Note that the response type is the non-standard
  253. 'application/x-json-stream' for lack of a better option.
  254. """
  255. if start is not None:
  256. start = float(start)
  257. if end is not None:
  258. end = float(end)
  259. if start is not None and end is not None:
  260. if start >= end:
  261. raise cherrypy.HTTPError("400 Bad Request",
  262. "start must precede end")
  263. streams = self.db.stream_list(path = path)
  264. if len(streams) != 1:
  265. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  266. @workaround_cp_bug_1200
  267. def content(start, end):
  268. # Note: disable chunked responses to see tracebacks from here.
  269. while True:
  270. (ints, restart) = self.db.stream_intervals(path, start, end)
  271. response = ''.join([ json.dumps(i) + "\r\n" for i in ints ])
  272. yield response
  273. if restart == 0:
  274. break
  275. start = restart
  276. return content(start, end)
  277. # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
  278. @cherrypy.expose
  279. @chunked_response
  280. @response_type("text/plain")
  281. def extract(self, path, start = None, end = None, count = False):
  282. """
  283. Extract data from backend database. Streams the resulting
  284. entries as ASCII text lines separated by newlines. This may
  285. make multiple requests to the nilmdb backend to avoid causing
  286. it to block for too long.
  287. Add count=True to return a count rather than actual data.
  288. """
  289. if start is not None:
  290. start = float(start)
  291. if end is not None:
  292. end = float(end)
  293. # Check parameters
  294. if start is not None and end is not None:
  295. if start >= end:
  296. raise cherrypy.HTTPError("400 Bad Request",
  297. "start must precede end")
  298. # Check path and get layout
  299. streams = self.db.stream_list(path = path)
  300. if len(streams) != 1:
  301. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  302. layout = streams[0][1]
  303. # Get formatter
  304. formatter = nilmdb.server.layout.Formatter(layout)
  305. @workaround_cp_bug_1200
  306. def content(start, end, count):
  307. # Note: disable chunked responses to see tracebacks from here.
  308. if count:
  309. matched = self.db.stream_extract(path, start, end, count)
  310. yield sprintf("%d\n", matched)
  311. return
  312. while True:
  313. (data, restart) = self.db.stream_extract(path, start, end)
  314. # Format the data and yield it
  315. yield formatter.format(data)
  316. if restart == 0:
  317. return
  318. start = restart
  319. return content(start, end, count)
  320. class Exiter(object):
  321. """App that exits the server, for testing"""
  322. @cherrypy.expose
  323. def index(self):
  324. cherrypy.response.headers['Content-Type'] = 'text/plain'
  325. def content():
  326. yield 'Exiting by request'
  327. raise SystemExit
  328. return content()
  329. index._cp_config = { 'response.stream': True }
  330. class Server(object):
  331. def __init__(self, db, host = '127.0.0.1', port = 8080,
  332. stoppable = False, # whether /exit URL exists
  333. embedded = True, # hide diagnostics and output, etc
  334. fast_shutdown = False, # don't wait for clients to disconn.
  335. force_traceback = False # include traceback in all errors
  336. ):
  337. # Save server version, just for verification during tests
  338. self.version = nilmdb.__version__
  339. self.embedded = embedded
  340. self.db = db
  341. if not getattr(db, "_thread_safe", None):
  342. raise KeyError("Database object " + str(db) + " doesn't claim "
  343. "to be thread safe. You should pass "
  344. "nilmdb.utils.serializer_proxy(NilmDB)(args) "
  345. "rather than NilmDB(args).")
  346. # Build up global server configuration
  347. cherrypy.config.update({
  348. 'server.socket_host': host,
  349. 'server.socket_port': port,
  350. 'engine.autoreload_on': False,
  351. 'server.max_request_body_size': 8*1024*1024,
  352. })
  353. if self.embedded:
  354. cherrypy.config.update({ 'environment': 'embedded' })
  355. # Build up application specific configuration
  356. app_config = {}
  357. app_config.update({
  358. 'error_page.default': self.json_error_page,
  359. })
  360. # Send a permissive Access-Control-Allow-Origin (CORS) header
  361. # with all responses so that browsers can send cross-domain
  362. # requests to this server.
  363. app_config.update({ 'response.headers.Access-Control-Allow-Origin':
  364. '*' })
  365. # Only allow GET and HEAD by default. Individual handlers
  366. # can override.
  367. app_config.update({ 'tools.allow.on': True,
  368. 'tools.allow.methods': ['GET', 'HEAD'] })
  369. # Send tracebacks in error responses. They're hidden by the
  370. # error_page function for client errors (code 400-499).
  371. app_config.update({ 'request.show_tracebacks' : True })
  372. self.force_traceback = force_traceback
  373. # Patch CherryPy error handler to never pad out error messages.
  374. # This isn't necessary, but then again, neither is padding the
  375. # error messages.
  376. cherrypy._cperror._ie_friendly_error_sizes = {}
  377. # Build up the application and mount it
  378. root = Root(self.db)
  379. root.stream = Stream(self.db)
  380. if stoppable:
  381. root.exit = Exiter()
  382. cherrypy.tree.apps = {}
  383. cherrypy.tree.mount(root, "/", config = { "/" : app_config })
  384. # Shutdowns normally wait for clients to disconnect. To speed
  385. # up tests, set fast_shutdown = True
  386. if fast_shutdown:
  387. # Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
  388. cherrypy.server.shutdown_timeout = 0.01
  389. else:
  390. cherrypy.server.shutdown_timeout = 5
  391. def json_error_page(self, status, message, traceback, version):
  392. """Return a custom error page in JSON so the client can parse it"""
  393. errordata = { "status" : status,
  394. "message" : message,
  395. "traceback" : traceback }
  396. # Don't send a traceback if the error was 400-499 (client's fault)
  397. try:
  398. code = int(status.split()[0])
  399. if not self.force_traceback:
  400. if code >= 400 and code <= 499:
  401. errordata["traceback"] = ""
  402. except Exception: # pragma: no cover
  403. pass
  404. # Override the response type, which was previously set to text/html
  405. cherrypy.serving.response.headers['Content-Type'] = (
  406. "application/json;charset=utf-8" )
  407. # Undo the HTML escaping that cherrypy's get_error_page function applies
  408. # (cherrypy issue 1135)
  409. for k, v in errordata.iteritems():
  410. v = v.replace("&lt;","<")
  411. v = v.replace("&gt;",">")
  412. v = v.replace("&amp;","&")
  413. errordata[k] = v
  414. return json.dumps(errordata, separators=(',',':'))
  415. def start(self, blocking = False, event = None):
  416. if not self.embedded: # pragma: no cover
  417. # Handle signals nicely
  418. if hasattr(cherrypy.engine, "signal_handler"):
  419. cherrypy.engine.signal_handler.subscribe()
  420. if hasattr(cherrypy.engine, "console_control_handler"):
  421. cherrypy.engine.console_control_handler.subscribe()
  422. # Cherrypy stupidly calls os._exit(70) when it can't bind the
  423. # port. At least try to print a reasonable error and continue
  424. # in this case, rather than just dying silently (as we would
  425. # otherwise do in embedded mode)
  426. real_exit = os._exit
  427. def fake_exit(code): # pragma: no cover
  428. if code == os.EX_SOFTWARE:
  429. fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
  430. else:
  431. real_exit(code)
  432. os._exit = fake_exit
  433. cherrypy.engine.start()
  434. os._exit = real_exit
  435. # Signal that the engine has started successfully
  436. if event is not None:
  437. event.set()
  438. if blocking:
  439. try:
  440. cherrypy.engine.wait(cherrypy.engine.states.EXITING,
  441. interval = 0.1, channel = 'main')
  442. except (KeyboardInterrupt, IOError): # pragma: no cover
  443. cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
  444. cherrypy.engine.exit()
  445. except SystemExit: # pragma: no cover
  446. cherrypy.engine.log('SystemExit raised: shutting down bus')
  447. cherrypy.engine.exit()
  448. raise
  449. def stop(self):
  450. cherrypy.engine.exit()