From 58c0ae72f683a00122377c3aec35499538f6d217 Mon Sep 17 00:00:00 2001 From: Jim Paris Date: Tue, 5 Mar 2013 11:54:29 -0500 Subject: [PATCH] Support application/json POST bodies as well as x-www-form-urlencoded --- nilmdb/client/client.py | 7 +++- nilmdb/client/httpclient.py | 28 ++++++++++---- nilmdb/server/server.py | 19 ++++++++++ tests/test_client.py | 76 +++++++++++++++++++------------------ tests/test_nilmdb.py | 17 +++++++++ 5 files changed, 100 insertions(+), 47 deletions(-) diff --git a/nilmdb/client/client.py b/nilmdb/client/client.py index 435a1e8..554f79a 100644 --- a/nilmdb/client/client.py +++ b/nilmdb/client/client.py @@ -21,8 +21,11 @@ def extract_timestamp(line): class Client(object): """Main client interface to the Nilm database.""" - def __init__(self, url): - self.http = nilmdb.client.httpclient.HTTPClient(url) + def __init__(self, url, post_json = False): + """Initialize client with given URL. If post_json is true, + POST requests are sent with Content-Type 'application/json' + instead of the default 'x-www-form-urlencoded'.""" + self.http = nilmdb.client.httpclient.HTTPClient(url, post_json) # __enter__/__exit__ allow this class to be a context manager def __enter__(self): diff --git a/nilmdb/client/httpclient.py b/nilmdb/client/httpclient.py index 2509b10..fc34e80 100644 --- a/nilmdb/client/httpclient.py +++ b/nilmdb/client/httpclient.py @@ -10,7 +10,7 @@ import requests class HTTPClient(object): """Class to manage and perform HTTP requests from the client""" - def __init__(self, baseurl = ""): + def __init__(self, baseurl = "", post_json = False): """If baseurl is supplied, all other functions that take a URL can be given a relative URL instead.""" # Verify / clean up URL @@ -26,6 +26,10 @@ class HTTPClient(object): # Saved response, so that tests can verify a few things. self._last_response = {} + # Whether to send application/json POST bodies (versus + # x-www-form-urlencoded) + self.post_json = post_json + def _handle_error(self, url, code, body): # Default variables for exception. We use the entire body as # the default message, in case we can't extract it from a JSON @@ -57,13 +61,14 @@ class HTTPClient(object): def close(self): self.session.close() - def _do_req(self, method, url, query_data, body_data, stream): + def _do_req(self, method, url, query_data, body_data, stream, headers): url = urlparse.urljoin(self.baseurl, url) try: response = self.session.request(method, url, params = query_data, data = body_data, - stream = stream) + stream = stream, + headers = headers) except requests.RequestException as e: raise ServerError(status = "502 Error", url = url, message = str(e.message)) @@ -77,12 +82,13 @@ class HTTPClient(object): return (response, False) # Normal versions that return data directly - def _req(self, method, url, query = None, body = None): + def _req(self, method, url, query = None, body = None, headers = None): """ Make a request and return the body data as a string or parsed JSON object, or raise an error if it contained an error. """ - (response, isjson) = self._do_req(method, url, query, body, False) + (response, isjson) = self._do_req(method, url, query, body, + stream = False, headers = headers) if isjson: return json.loads(response.content) return response.content @@ -93,20 +99,26 @@ class HTTPClient(object): def post(self, url, params = None): """Simple POST (parameters in body)""" - return self._req("POST", url, None, params) + if self.post_json: + return self._req("POST", url, None, + json.dumps(params), + { 'Content-type': 'application/json' }) + else: + return self._req("POST", url, None, params) def put(self, url, data, params = None): """Simple PUT (parameters in URL, data in body)""" return self._req("PUT", url, params, data) # Generator versions that return data one line at a time. - def _req_gen(self, method, url, query = None, body = None): + def _req_gen(self, method, url, query = None, body = None, headers = None): """ Make a request and return a generator that gives back strings or JSON decoded lines of the body data, or raise an error if it contained an eror. """ - (response, isjson) = self._do_req(method, url, query, body, True) + (response, isjson) = self._do_req(method, url, query, body, + stream = True, headers = headers) for line in response.iter_lines(): if isjson: yield json.loads(line) diff --git a/nilmdb/server/server.py b/nilmdb/server/server.py index 2861c16..7ce9a67 100644 --- a/nilmdb/server/server.py +++ b/nilmdb/server/server.py @@ -117,6 +117,14 @@ def CORS_allow(methods): cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow) +# Helper for json_in tool to process JSON data into normal request +# parameters. +def json_to_request_params(body): + cherrypy.lib.jsontools.json_processor(body) + if not isinstance(cherrypy.request.json, dict): + raise cherrypy.HTTPError(415) + cherrypy.request.params.update(cherrypy.request.json) + # CherryPy apps class Root(NilmApp): """Root application for NILM database""" @@ -175,6 +183,7 @@ class Stream(NilmApp): # /stream/create?path=/newton/prep&layout=PrepData @cherrypy.expose + @cherrypy.tools.json_in() @cherrypy.tools.json_out() @exception_to_httperror(NilmDBError, ValueError) @cherrypy.tools.CORS_allow(methods = ["POST"]) @@ -186,6 +195,7 @@ class Stream(NilmApp): # /stream/destroy?path=/newton/prep @cherrypy.expose + @cherrypy.tools.json_in() @cherrypy.tools.json_out() @exception_to_httperror(NilmDBError) @cherrypy.tools.CORS_allow(methods = ["POST"]) @@ -219,6 +229,7 @@ class Stream(NilmApp): # /stream/set_metadata?path=/newton/prep&data= @cherrypy.expose + @cherrypy.tools.json_in() @cherrypy.tools.json_out() @exception_to_httperror(NilmDBError, LookupError, TypeError) @cherrypy.tools.CORS_allow(methods = ["POST"]) @@ -231,6 +242,7 @@ class Stream(NilmApp): # /stream/update_metadata?path=/newton/prep&data= @cherrypy.expose + @cherrypy.tools.json_in() @cherrypy.tools.json_out() @exception_to_httperror(NilmDBError, LookupError, TypeError) @cherrypy.tools.CORS_allow(methods = ["POST"]) @@ -297,6 +309,7 @@ class Stream(NilmApp): # /stream/remove?path=/newton/prep # /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0 @cherrypy.expose + @cherrypy.tools.json_in() @cherrypy.tools.json_out() @exception_to_httperror(NilmDBError) @cherrypy.tools.CORS_allow(methods = ["POST"]) @@ -463,6 +476,12 @@ class Server(object): app_config.update({ 'tools.CORS_allow.on': True, 'tools.CORS_allow.methods': ['GET', 'HEAD'] }) + # Configure the 'json_in' tool to also allow other content-types + # (like x-www-form-urlencoded), and to treat JSON as a dict that + # fills requests.param. + app_config.update({ 'tools.json_in.force': False, + 'tools.json_in.processor': json_to_request_params }) + # Send tracebacks in error responses. They're hidden by the # error_page function for client errors (code 400-499). app_config.update({ 'request.show_tracebacks' : True }) diff --git a/tests/test_client.py b/tests/test_client.py index cfc584f..8d89bbf 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -357,43 +357,45 @@ class TestClient(object): client.close() def test_client_08_unicode(self): - # Basic Unicode tests - client = nilmdb.Client(url = testurl) - - # Delete streams that exist - for stream in client.stream_list(): - client.stream_destroy(stream[0]) - - # Database is empty - eq_(client.stream_list(), []) - - # Create Unicode stream, match it - raw = [ u"/düsseldorf/raw", u"uint16_6" ] - prep = [ u"/düsseldorf/prep", u"uint16_6" ] - client.stream_create(*raw) - eq_(client.stream_list(), [raw]) - eq_(client.stream_list(layout=raw[1]), [raw]) - eq_(client.stream_list(path=raw[0]), [raw]) - client.stream_create(*prep) - eq_(client.stream_list(), [prep, raw]) - - # Set / get metadata with Unicode keys and values - eq_(client.stream_get_metadata(raw[0]), {}) - eq_(client.stream_get_metadata(prep[0]), {}) - meta1 = { u"alpha": u"α", - u"β": u"beta" } - meta2 = { u"alpha": u"α" } - meta3 = { u"β": u"beta" } - client.stream_set_metadata(prep[0], meta1) - client.stream_update_metadata(prep[0], {}) - client.stream_update_metadata(raw[0], meta2) - client.stream_update_metadata(raw[0], meta3) - eq_(client.stream_get_metadata(prep[0]), meta1) - eq_(client.stream_get_metadata(raw[0]), meta1) - eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2) - eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1) - - client.close() + # Try both with and without posting JSON + for post_json in (False, True): + # Basic Unicode tests + client = nilmdb.Client(url = testurl, post_json = post_json) + + # Delete streams that exist + for stream in client.stream_list(): + client.stream_destroy(stream[0]) + + # Database is empty + eq_(client.stream_list(), []) + + # Create Unicode stream, match it + raw = [ u"/düsseldorf/raw", u"uint16_6" ] + prep = [ u"/düsseldorf/prep", u"uint16_6" ] + client.stream_create(*raw) + eq_(client.stream_list(), [raw]) + eq_(client.stream_list(layout=raw[1]), [raw]) + eq_(client.stream_list(path=raw[0]), [raw]) + client.stream_create(*prep) + eq_(client.stream_list(), [prep, raw]) + + # Set / get metadata with Unicode keys and values + eq_(client.stream_get_metadata(raw[0]), {}) + eq_(client.stream_get_metadata(prep[0]), {}) + meta1 = { u"alpha": u"α", + u"β": u"beta" } + meta2 = { u"alpha": u"α" } + meta3 = { u"β": u"beta" } + client.stream_set_metadata(prep[0], meta1) + client.stream_update_metadata(prep[0], {}) + client.stream_update_metadata(raw[0], meta2) + client.stream_update_metadata(raw[0], meta3) + eq_(client.stream_get_metadata(prep[0]), meta1) + eq_(client.stream_get_metadata(raw[0]), meta1) + eq_(client.stream_get_metadata(raw[0], [ "alpha" ]), meta2) + eq_(client.stream_get_metadata(raw[0], [ "alpha", "β" ]), meta1) + + client.close() def test_client_09_closing(self): # Make sure we actually close sockets correctly. New diff --git a/tests/test_nilmdb.py b/tests/test_nilmdb.py index 9a7161f..8e669e5 100644 --- a/tests/test_nilmdb.py +++ b/tests/test_nilmdb.py @@ -237,3 +237,20 @@ class TestServer(object): "header in response:\n", r.headers) eq_(r.headers["access-control-allow-methods"], "GET, HEAD") eq_(r.headers["access-control-allow-headers"], "X-Custom") + + def test_post_bodies(self): + # Test JSON post bodies + r = requests.post("http://127.0.0.1:32180/stream/set_metadata", + headers = { "Content-Type": "application/json" }, + data = '{"hello": 1}') + eq_(r.status_code, 404) # wrong parameters + + r = requests.post("http://127.0.0.1:32180/stream/set_metadata", + headers = { "Content-Type": "application/json" }, + data = '["hello"]') + eq_(r.status_code, 415) # not a dict + + r = requests.post("http://127.0.0.1:32180/stream/set_metadata", + headers = { "Content-Type": "application/json" }, + data = '[hello]') + eq_(r.status_code, 400) # badly formatted JSON