You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

503 lines
18 KiB

  1. """CherryPy-based server for accessing NILM database via HTTP"""
  2. # Need absolute_import so that "import nilmdb" won't pull in
  3. # nilmdb.py, but will pull the nilmdb module instead.
  4. from __future__ import absolute_import
  5. import nilmdb
  6. from nilmdb.utils.printf import *
  7. from nilmdb.server.errors import NilmDBError
  8. import cherrypy
  9. import sys
  10. import os
  11. import simplejson as json
  12. import decorator
  13. import traceback
  14. import psutil
  15. try:
  16. cherrypy.tools.json_out
  17. except: # pragma: no cover
  18. sys.stderr.write("Cherrypy 3.2+ required\n")
  19. sys.exit(1)
  20. class NilmApp(object):
  21. def __init__(self, db):
  22. self.db = db
  23. # Decorators
  24. def chunked_response(func):
  25. """Decorator to enable chunked responses."""
  26. # Set this to False to get better tracebacks from some requests
  27. # (/stream/extract, /stream/intervals).
  28. func._cp_config = { 'response.stream': True }
  29. return func
  30. def response_type(content_type):
  31. """Return a decorator-generating function that sets the
  32. response type to the specified string."""
  33. def wrapper(func, *args, **kwargs):
  34. cherrypy.response.headers['Content-Type'] = content_type
  35. return func(*args, **kwargs)
  36. return decorator.decorator(wrapper)
  37. @decorator.decorator
  38. def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
  39. """Decorator to work around CherryPy bug #1200 in a response
  40. generator.
  41. Even if chunked responses are disabled, LookupError or
  42. UnicodeError exceptions may still be swallowed by CherryPy due to
  43. bug #1200. This throws them as generic Exceptions instead so that
  44. they make it through.
  45. """
  46. try:
  47. for val in func(*args, **kwargs):
  48. yield val
  49. except (LookupError, UnicodeError):
  50. raise Exception("bug workaround; real exception is:\n" +
  51. traceback.format_exc())
  52. def exception_to_httperror(*expected):
  53. """Return a decorator-generating function that catches expected
  54. errors and throws a HTTPError describing it instead.
  55. @exception_to_httperror(NilmDBError, ValueError)
  56. def foo():
  57. pass
  58. """
  59. def wrapper(func, *args, **kwargs):
  60. try:
  61. return func(*args, **kwargs)
  62. except expected as e:
  63. message = sprintf("%s", str(e))
  64. raise cherrypy.HTTPError("400 Bad Request", message)
  65. # We need to preserve the function's argspecs for CherryPy to
  66. # handle argument errors correctly. Decorator.decorator takes
  67. # care of that.
  68. return decorator.decorator(wrapper)
  69. # CherryPy apps
  70. class Root(NilmApp):
  71. """Root application for NILM database"""
  72. def __init__(self, db):
  73. super(Root, self).__init__(db)
  74. # /
  75. @cherrypy.expose
  76. def index(self):
  77. raise cherrypy.NotFound()
  78. # /favicon.ico
  79. @cherrypy.expose
  80. def favicon_ico(self):
  81. raise cherrypy.NotFound()
  82. # /version
  83. @cherrypy.expose
  84. @cherrypy.tools.json_out()
  85. def version(self):
  86. return nilmdb.__version__
  87. # /dbinfo
  88. @cherrypy.expose
  89. @cherrypy.tools.json_out()
  90. def dbinfo(self):
  91. """Return a dictionary with the database path,
  92. size of the database in bytes, and free disk space in bytes"""
  93. path = self.db.get_basepath()
  94. return { "path": path,
  95. "size": nilmdb.utils.du(path),
  96. "free": psutil.disk_usage(path).free }
  97. class Stream(NilmApp):
  98. """Stream-specific operations"""
  99. # /stream/list
  100. # /stream/list?layout=PrepData
  101. # /stream/list?path=/newton/prep
  102. @cherrypy.expose
  103. @cherrypy.tools.json_out()
  104. def list(self, path = None, layout = None):
  105. """List all streams in the database. With optional path or
  106. layout parameter, just list streams that match the given path
  107. or layout"""
  108. return self.db.stream_list(path, layout)
  109. # /stream/create?path=/newton/prep&layout=PrepData
  110. @cherrypy.expose
  111. @cherrypy.tools.json_out()
  112. @exception_to_httperror(NilmDBError, ValueError)
  113. def create(self, path, layout):
  114. """Create a new stream in the database. Provide path
  115. and one of the nilmdb.layout.layouts keys.
  116. """
  117. return self.db.stream_create(path, layout)
  118. # /stream/destroy?path=/newton/prep
  119. @cherrypy.expose
  120. @cherrypy.tools.json_out()
  121. @exception_to_httperror(NilmDBError)
  122. def destroy(self, path):
  123. """Delete a stream and its associated data."""
  124. return self.db.stream_destroy(path)
  125. # /stream/get_metadata?path=/newton/prep
  126. # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
  127. @cherrypy.expose
  128. @cherrypy.tools.json_out()
  129. def get_metadata(self, path, key=None):
  130. """Get metadata for the named stream. If optional
  131. key parameters are specified, only return metadata
  132. matching the given keys."""
  133. try:
  134. data = self.db.stream_get_metadata(path)
  135. except nilmdb.server.nilmdb.StreamError as e:
  136. raise cherrypy.HTTPError("404 Not Found", e.message)
  137. if key is None: # If no keys specified, return them all
  138. key = data.keys()
  139. elif not isinstance(key, list):
  140. key = [ key ]
  141. result = {}
  142. for k in key:
  143. if k in data:
  144. result[k] = data[k]
  145. else: # Return "None" for keys with no matching value
  146. result[k] = None
  147. return result
  148. # /stream/set_metadata?path=/newton/prep&data=<json>
  149. @cherrypy.expose
  150. @cherrypy.tools.json_out()
  151. @exception_to_httperror(NilmDBError, LookupError, TypeError)
  152. def set_metadata(self, path, data):
  153. """Set metadata for the named stream, replacing any
  154. existing metadata. Data should be a json-encoded
  155. dictionary"""
  156. data_dict = json.loads(data)
  157. self.db.stream_set_metadata(path, data_dict)
  158. # /stream/update_metadata?path=/newton/prep&data=<json>
  159. @cherrypy.expose
  160. @cherrypy.tools.json_out()
  161. @exception_to_httperror(NilmDBError, LookupError, TypeError)
  162. def update_metadata(self, path, data):
  163. """Update metadata for the named stream. Data
  164. should be a json-encoded dictionary"""
  165. data_dict = json.loads(data)
  166. self.db.stream_update_metadata(path, data_dict)
  167. # /stream/insert?path=/newton/prep
  168. @cherrypy.expose
  169. @cherrypy.tools.json_out()
  170. #@cherrypy.tools.disable_prb()
  171. def insert(self, path, start, end):
  172. """
  173. Insert new data into the database. Provide textual data
  174. (matching the path's layout) as a HTTP PUT.
  175. """
  176. # Important that we always read the input before throwing any
  177. # errors, to keep lengths happy for persistent connections.
  178. # However, CherryPy 3.2.2 has a bug where this fails for GET
  179. # requests, so catch that. (issue #1134)
  180. try:
  181. body = cherrypy.request.body.read()
  182. except TypeError:
  183. raise cherrypy.HTTPError("400 Bad Request", "No request body")
  184. # Check path and get layout
  185. streams = self.db.stream_list(path = path)
  186. if len(streams) != 1:
  187. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  188. layout = streams[0][1]
  189. # Parse the input data
  190. try:
  191. parser = nilmdb.server.layout.Parser(layout)
  192. parser.parse(body)
  193. except nilmdb.server.layout.ParserError as e:
  194. raise cherrypy.HTTPError("400 Bad Request",
  195. "error parsing input data: " +
  196. e.message)
  197. # Check limits
  198. start = float(start)
  199. end = float(end)
  200. if start >= end:
  201. raise cherrypy.HTTPError("400 Bad Request",
  202. "start must precede end")
  203. if parser.min_timestamp is not None and parser.min_timestamp < start:
  204. raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
  205. repr(parser.min_timestamp) +
  206. " < start time " + repr(start))
  207. if parser.max_timestamp is not None and parser.max_timestamp >= end:
  208. raise cherrypy.HTTPError("400 Bad Request", "Data timestamp " +
  209. repr(parser.max_timestamp) +
  210. " >= end time " + repr(end))
  211. # Now do the nilmdb insert, passing it the parser full of data.
  212. try:
  213. self.db.stream_insert(path, start, end, parser.data)
  214. except NilmDBError as e:
  215. raise cherrypy.HTTPError("400 Bad Request", e.message)
  216. # Done
  217. return
  218. # /stream/remove?path=/newton/prep
  219. # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
  220. @cherrypy.expose
  221. @cherrypy.tools.json_out()
  222. @exception_to_httperror(NilmDBError)
  223. def remove(self, path, start = None, end = None):
  224. """
  225. Remove data from the backend database. Removes all data in
  226. the interval [start, end). Returns the number of data points
  227. removed.
  228. """
  229. if start is not None:
  230. start = float(start)
  231. if end is not None:
  232. end = float(end)
  233. if start is not None and end is not None:
  234. if start >= end:
  235. raise cherrypy.HTTPError("400 Bad Request",
  236. "start must precede end")
  237. return self.db.stream_remove(path, start, end)
  238. # /stream/intervals?path=/newton/prep
  239. # /stream/intervals?path=/newton/prep&start=1234567890.0&end=1234567899.0
  240. @cherrypy.expose
  241. @chunked_response
  242. @response_type("text/plain")
  243. def intervals(self, path, start = None, end = None):
  244. """
  245. Get intervals from backend database. Streams the resulting
  246. intervals as JSON strings separated by newlines. This may
  247. make multiple requests to the nilmdb backend to avoid causing
  248. it to block for too long.
  249. Note that the response type is set to 'text/plain' even
  250. though we're sending back JSON; this is because we're not
  251. really returning a single JSON object.
  252. """
  253. if start is not None:
  254. start = float(start)
  255. if end is not None:
  256. end = float(end)
  257. if start is not None and end is not None:
  258. if start >= end:
  259. raise cherrypy.HTTPError("400 Bad Request",
  260. "start must precede end")
  261. streams = self.db.stream_list(path = path)
  262. if len(streams) != 1:
  263. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  264. @workaround_cp_bug_1200
  265. def content(start, end):
  266. # Note: disable chunked responses to see tracebacks from here.
  267. while True:
  268. (intervals, restart) = self.db.stream_intervals(path, start, end)
  269. response = ''.join([ json.dumps(i) + "\n" for i in intervals ])
  270. yield response
  271. if restart == 0:
  272. break
  273. start = restart
  274. return content(start, end)
  275. # /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
  276. @cherrypy.expose
  277. @chunked_response
  278. @response_type("text/plain")
  279. def extract(self, path, start = None, end = None, count = False):
  280. """
  281. Extract data from backend database. Streams the resulting
  282. entries as ASCII text lines separated by newlines. This may
  283. make multiple requests to the nilmdb backend to avoid causing
  284. it to block for too long.
  285. Add count=True to return a count rather than actual data.
  286. """
  287. if start is not None:
  288. start = float(start)
  289. if end is not None:
  290. end = float(end)
  291. # Check parameters
  292. if start is not None and end is not None:
  293. if start >= end:
  294. raise cherrypy.HTTPError("400 Bad Request",
  295. "start must precede end")
  296. # Check path and get layout
  297. streams = self.db.stream_list(path = path)
  298. if len(streams) != 1:
  299. raise cherrypy.HTTPError("404 Not Found", "No such stream")
  300. layout = streams[0][1]
  301. # Get formatter
  302. formatter = nilmdb.server.layout.Formatter(layout)
  303. @workaround_cp_bug_1200
  304. def content(start, end, count):
  305. # Note: disable chunked responses to see tracebacks from here.
  306. if count:
  307. matched = self.db.stream_extract(path, start, end, count)
  308. yield sprintf("%d\n", matched)
  309. return
  310. while True:
  311. (data, restart) = self.db.stream_extract(path, start, end)
  312. # Format the data and yield it
  313. yield formatter.format(data)
  314. if restart == 0:
  315. return
  316. start = restart
  317. return content(start, end, count)
  318. class Exiter(object):
  319. """App that exits the server, for testing"""
  320. @cherrypy.expose
  321. def index(self):
  322. cherrypy.response.headers['Content-Type'] = 'text/plain'
  323. def content():
  324. yield 'Exiting by request'
  325. raise SystemExit
  326. return content()
  327. index._cp_config = { 'response.stream': True }
  328. class Server(object):
  329. def __init__(self, db, host = '127.0.0.1', port = 8080,
  330. stoppable = False, # whether /exit URL exists
  331. embedded = True, # hide diagnostics and output, etc
  332. fast_shutdown = False, # don't wait for clients to disconn.
  333. force_traceback = False # include traceback in all errors
  334. ):
  335. # Save server version, just for verification during tests
  336. self.version = nilmdb.__version__
  337. # Need to wrap DB object in a serializer because we'll call
  338. # into it from separate threads.
  339. self.embedded = embedded
  340. self.db = nilmdb.utils.Serializer(db)
  341. # Build up global server configuration
  342. cherrypy.config.update({
  343. 'server.socket_host': host,
  344. 'server.socket_port': port,
  345. 'engine.autoreload_on': False,
  346. 'server.max_request_body_size': 4*1024*1024,
  347. })
  348. if self.embedded:
  349. cherrypy.config.update({ 'environment': 'embedded' })
  350. # Build up application specific configuration
  351. app_config = {}
  352. app_config.update({
  353. 'error_page.default': self.json_error_page,
  354. })
  355. # Send a permissive Access-Control-Allow-Origin (CORS) header
  356. # with all responses so that browsers can send cross-domain
  357. # requests to this server.
  358. app_config.update({ 'response.headers.Access-Control-Allow-Origin':
  359. '*' })
  360. # Send tracebacks in error responses. They're hidden by the
  361. # error_page function for client errors (code 400-499).
  362. app_config.update({ 'request.show_tracebacks' : True })
  363. self.force_traceback = force_traceback
  364. # Patch CherryPy error handler to never pad out error messages.
  365. # This isn't necessary, but then again, neither is padding the
  366. # error messages.
  367. cherrypy._cperror._ie_friendly_error_sizes = {}
  368. # Build up the application and mount it
  369. root = Root(self.db)
  370. root.stream = Stream(self.db)
  371. if stoppable:
  372. root.exit = Exiter()
  373. cherrypy.tree.apps = {}
  374. cherrypy.tree.mount(root, "/", config = { "/" : app_config })
  375. # Shutdowns normally wait for clients to disconnect. To speed
  376. # up tests, set fast_shutdown = True
  377. if fast_shutdown:
  378. # Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
  379. cherrypy.server.shutdown_timeout = 0.01
  380. else:
  381. cherrypy.server.shutdown_timeout = 5
  382. def json_error_page(self, status, message, traceback, version):
  383. """Return a custom error page in JSON so the client can parse it"""
  384. errordata = { "status" : status,
  385. "message" : message,
  386. "traceback" : traceback }
  387. # Don't send a traceback if the error was 400-499 (client's fault)
  388. try:
  389. code = int(status.split()[0])
  390. if not self.force_traceback:
  391. if code >= 400 and code <= 499:
  392. errordata["traceback"] = ""
  393. except Exception: # pragma: no cover
  394. pass
  395. # Override the response type, which was previously set to text/html
  396. cherrypy.serving.response.headers['Content-Type'] = (
  397. "application/json;charset=utf-8" )
  398. # Undo the HTML escaping that cherrypy's get_error_page function applies
  399. # (cherrypy issue 1135)
  400. for k, v in errordata.iteritems():
  401. v = v.replace("&lt;","<")
  402. v = v.replace("&gt;",">")
  403. v = v.replace("&amp;","&")
  404. errordata[k] = v
  405. return json.dumps(errordata, separators=(',',':'))
  406. def start(self, blocking = False, event = None):
  407. if not self.embedded: # pragma: no cover
  408. # Handle signals nicely
  409. if hasattr(cherrypy.engine, "signal_handler"):
  410. cherrypy.engine.signal_handler.subscribe()
  411. if hasattr(cherrypy.engine, "console_control_handler"):
  412. cherrypy.engine.console_control_handler.subscribe()
  413. # Cherrypy stupidly calls os._exit(70) when it can't bind the
  414. # port. At least try to print a reasonable error and continue
  415. # in this case, rather than just dying silently (as we would
  416. # otherwise do in embedded mode)
  417. real_exit = os._exit
  418. def fake_exit(code): # pragma: no cover
  419. if code == os.EX_SOFTWARE:
  420. fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
  421. else:
  422. real_exit(code)
  423. os._exit = fake_exit
  424. cherrypy.engine.start()
  425. os._exit = real_exit
  426. # Signal that the engine has started successfully
  427. if event is not None:
  428. event.set()
  429. if blocking:
  430. try:
  431. cherrypy.engine.wait(cherrypy.engine.states.EXITING,
  432. interval = 0.1, channel = 'main')
  433. except (KeyboardInterrupt, IOError): # pragma: no cover
  434. cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
  435. cherrypy.engine.exit()
  436. except SystemExit: # pragma: no cover
  437. cherrypy.engine.log('SystemExit raised: shutting down bus')
  438. cherrypy.engine.exit()
  439. raise
  440. def stop(self):
  441. cherrypy.engine.exit()