This is a pretty big change that will render existing clients unable to modify the database, but it's important that we use POST or PUT instead of GET for anything that may change state, in case this is ever put behind a cache.tags/nilmdb-1.2
@@ -73,7 +73,7 @@ class Client(object): | |||
"path": path, | |||
"data": self._json_param(data) | |||
} | |||
return self.http.get("stream/set_metadata", params) | |||
return self.http.post("stream/set_metadata", params) | |||
def stream_update_metadata(self, path, data): | |||
"""Update stream metadata from a dictionary""" | |||
@@ -81,18 +81,18 @@ class Client(object): | |||
"path": path, | |||
"data": self._json_param(data) | |||
} | |||
return self.http.get("stream/update_metadata", params) | |||
return self.http.post("stream/update_metadata", params) | |||
def stream_create(self, path, layout): | |||
"""Create a new stream""" | |||
params = { "path": path, | |||
"layout" : layout } | |||
return self.http.get("stream/create", params) | |||
return self.http.post("stream/create", params) | |||
def stream_destroy(self, path): | |||
"""Delete stream and its contents""" | |||
params = { "path": path } | |||
return self.http.get("stream/destroy", params) | |||
return self.http.post("stream/destroy", params) | |||
def stream_remove(self, path, start = None, end = None): | |||
"""Remove data from the specified time range""" | |||
@@ -103,7 +103,7 @@ class Client(object): | |||
params["start"] = float_to_string(start) | |||
if end is not None: | |||
params["end"] = float_to_string(end) | |||
return self.http.get("stream/remove", params) | |||
return self.http.post("stream/remove", params) | |||
@contextlib.contextmanager | |||
def stream_insert_context(self, path, start = None, end = None): | |||
@@ -40,11 +40,16 @@ class HTTPClient(object): | |||
self.lock = HTTPClientLock() | |||
self._setup_url() | |||
def _setup_url(self, url = "", params = ""): | |||
def _setup_url(self, url = "", params = "", post_params = False): | |||
url = urlparse.urljoin(self.baseurl, url) | |||
self.curl.setopt(pycurl.POST, 0) | |||
if params: | |||
url = urlparse.urljoin( | |||
url, "?" + nilmdb.utils.urllib.urlencode(params)) | |||
param_string = nilmdb.utils.urllib.urlencode(params) | |||
if post_params: | |||
self.curl.setopt(pycurl.POSTFIELDS, param_string) | |||
self.curl.setopt(pycurl.POST, 1) | |||
else: | |||
url = urlparse.urljoin(url, '?' + param_string) | |||
self.curl.setopt(pycurl.URL, url) | |||
self.url = url | |||
@@ -79,12 +84,12 @@ class HTTPClient(object): | |||
else: | |||
raise Error(**args) | |||
def _req_generator(self, url, params): | |||
def _req_generator(self, url, params, post_params = False): | |||
""" | |||
Like self._req(), but returns a generator that spits out | |||
arbitrary-sized chunks of the resulting data. | |||
""" | |||
self._setup_url(url, params) | |||
self._setup_url(url, params, post_params) | |||
error_body = "" | |||
self._headers = "" | |||
self._req_status = None | |||
@@ -119,12 +124,12 @@ class HTTPClient(object): | |||
# Raise an exception if there was an error | |||
self._check_error(error_body) | |||
def _req(self, url, params): | |||
def _req(self, url, params, post_params = False): | |||
""" | |||
GET or POST that returns raw data. Returns the body | |||
data as a string, or raises an error if it contained an error. | |||
""" | |||
self._setup_url(url, params) | |||
self._setup_url(url, params, post_params) | |||
body = cStringIO.StringIO() | |||
self.curl.setopt(pycurl.WRITEFUNCTION, body.write) | |||
self._headers = "" | |||
@@ -170,21 +175,28 @@ class HTTPClient(object): | |||
# Normal versions that return data directly | |||
def get(self, url, params = None, retjson = True): | |||
"""Simple GET""" | |||
"""Simple GET (parameters in URL)""" | |||
with self.lock: | |||
self.curl.setopt(pycurl.UPLOAD, 0) | |||
self.curl.setopt(pycurl.READFUNCTION, lambda: None) | |||
return self._maybe_json(retjson, self._req(url, params, False)) | |||
def post(self, url, params = None, retjson = True): | |||
"""Simple POST (parameters in body)""" | |||
with self.lock: | |||
self.curl.setopt(pycurl.UPLOAD, 0) | |||
self.curl.setopt(pycurl.READFUNCTION, lambda: None) | |||
return self._maybe_json(retjson, self._req(url, params)) | |||
return self._maybe_json(retjson, self._req(url, params, True)) | |||
def put(self, url, postdata, params = None, retjson = True): | |||
"""Simple PUT""" | |||
def put(self, url, data, params = None, retjson = True): | |||
"""Simple PUT (parameters in URL, data in body)""" | |||
with self.lock: | |||
self.curl.setopt(pycurl.UPLOAD, 1) | |||
data = cStringIO.StringIO(postdata) | |||
data = cStringIO.StringIO(data) | |||
self.curl.setopt(pycurl.READFUNCTION, data.read) | |||
return self._maybe_json(retjson, self._req(url, params)) | |||
return self._maybe_json(retjson, self._req(url, params, False)) | |||
# Generator versions that return data one line at a time | |||
# Generator versions that return data one line at a time. | |||
def get_gen(self, url, params = None, retjson = True): | |||
"""Simple GET that yields one resonse line at a time""" | |||
with self.lock: | |||
@@ -193,11 +205,5 @@ class HTTPClient(object): | |||
for line in self._iterate_lines(self._req_generator(url, params)): | |||
yield self._maybe_json(retjson, line) | |||
def put_gen(self, url, postdata, params = None, retjson = True): | |||
"""Simple PUT that yields one response line at a time""" | |||
with self.lock: | |||
self.curl.setopt(pycurl.UPLOAD, 1) | |||
data = cStringIO.StringIO(postdata) | |||
self.curl.setopt(pycurl.READFUNCTION, data.read) | |||
for line in self._iterate_lines(self._req_generator(url, params)): | |||
yield self._maybe_json(retjson, line) | |||
# Not much use for a POST or PUT generator, since they don't | |||
# return much data. |
@@ -71,6 +71,17 @@ def exception_to_httperror(*expected): | |||
# care of that. | |||
return decorator.decorator(wrapper) | |||
# Custom Cherrypy tools | |||
def allow_methods(methods): | |||
method = cherrypy.request.method.upper() | |||
if method not in methods: | |||
if method in cherrypy.request.methods_with_bodies: | |||
cherrypy.request.body.read() | |||
allowed = ', '.join(methods) | |||
cherrypy.response.headers['Allow'] = allowed | |||
raise cherrypy.HTTPError(405, method + " not allowed; use " + allowed) | |||
cherrypy.tools.allow_methods = cherrypy.Tool('before_handler', allow_methods) | |||
# CherryPy apps | |||
class Root(NilmApp): | |||
"""Root application for NILM database""" | |||
@@ -123,6 +134,7 @@ class Stream(NilmApp): | |||
@cherrypy.expose | |||
@cherrypy.tools.json_out() | |||
@exception_to_httperror(NilmDBError, ValueError) | |||
@cherrypy.tools.allow_methods(methods = ["POST"]) | |||
def create(self, path, layout): | |||
"""Create a new stream in the database. Provide path | |||
and one of the nilmdb.layout.layouts keys. | |||
@@ -133,6 +145,7 @@ class Stream(NilmApp): | |||
@cherrypy.expose | |||
@cherrypy.tools.json_out() | |||
@exception_to_httperror(NilmDBError) | |||
@cherrypy.tools.allow_methods(methods = ["POST"]) | |||
def destroy(self, path): | |||
"""Delete a stream and its associated data.""" | |||
return self.db.stream_destroy(path) | |||
@@ -165,6 +178,7 @@ class Stream(NilmApp): | |||
@cherrypy.expose | |||
@cherrypy.tools.json_out() | |||
@exception_to_httperror(NilmDBError, LookupError, TypeError) | |||
@cherrypy.tools.allow_methods(methods = ["POST"]) | |||
def set_metadata(self, path, data): | |||
"""Set metadata for the named stream, replacing any | |||
existing metadata. Data should be a json-encoded | |||
@@ -176,6 +190,7 @@ class Stream(NilmApp): | |||
@cherrypy.expose | |||
@cherrypy.tools.json_out() | |||
@exception_to_httperror(NilmDBError, LookupError, TypeError) | |||
@cherrypy.tools.allow_methods(methods = ["POST"]) | |||
def update_metadata(self, path, data): | |||
"""Update metadata for the named stream. Data | |||
should be a json-encoded dictionary""" | |||
@@ -185,6 +200,7 @@ class Stream(NilmApp): | |||
# /stream/insert?path=/newton/prep | |||
@cherrypy.expose | |||
@cherrypy.tools.json_out() | |||
@cherrypy.tools.allow_methods(methods = ["PUT"]) | |||
def insert(self, path, start, end): | |||
""" | |||
Insert new data into the database. Provide textual data | |||
@@ -192,12 +208,9 @@ class Stream(NilmApp): | |||
""" | |||
# Important that we always read the input before throwing any | |||
# errors, to keep lengths happy for persistent connections. | |||
# However, CherryPy 3.2.2 has a bug where this fails for GET | |||
# requests, so catch that. (issue #1134) | |||
try: | |||
body = cherrypy.request.body.read() | |||
except TypeError: | |||
raise cherrypy.HTTPError("400 Bad Request", "No request body") | |||
# Note that CherryPy 3.2.2 has a bug where this fails for GET | |||
# requests, if we ever want to handle those (issue #1134) | |||
body = cherrypy.request.body.read() | |||
# Check path and get layout | |||
streams = self.db.stream_list(path = path) | |||
@@ -243,6 +256,7 @@ class Stream(NilmApp): | |||
@cherrypy.expose | |||
@cherrypy.tools.json_out() | |||
@exception_to_httperror(NilmDBError) | |||
@cherrypy.tools.allow_methods(methods = ["POST"]) | |||
def remove(self, path, start = None, end = None): | |||
""" | |||
Remove data from the backend database. Removes all data in | |||
@@ -404,6 +418,11 @@ class Server(object): | |||
app_config.update({ 'response.headers.Access-Control-Allow-Origin': | |||
'*' }) | |||
# Only allow GET and HEAD by default. Individual handlers | |||
# can override. | |||
app_config.update({ 'tools.allow_methods.on': True, | |||
'tools.allow_methods.methods': ['GET', 'HEAD'] }) | |||
# Send tracebacks in error responses. They're hidden by the | |||
# error_page function for client errors (code 400-499). | |||
app_config.update({ 'request.show_tracebacks' : True }) | |||
@@ -97,6 +97,15 @@ class TestClient(object): | |||
with assert_raises(ClientError): | |||
client.stream_create("/newton/prep", "NoSuchLayout") | |||
# Bad method types | |||
with assert_raises(ClientError): | |||
client.http.put("/stream/list","") | |||
# Try a bunch of times to make sure the request body is getting consumed | |||
for x in range(10): | |||
with assert_raises(ClientError): | |||
client.http.post("/stream/list") | |||
client = nilmdb.Client(url = testurl) | |||
# Create three streams | |||
client.stream_create("/newton/prep", "PrepData") | |||
client.stream_create("/newton/raw", "RawData") | |||
@@ -312,14 +321,6 @@ class TestClient(object): | |||
for (a, b) in itertools.izip(aa, bb): | |||
eq_(json.loads(a), b) | |||
# Check PUT with generator out | |||
with assert_raises(ClientError) as e: | |||
client.http.put_gen("stream/insert", "", | |||
{ "path": "/newton/prep", | |||
"start": 0, "end": 0 }).next() | |||
in_("400 Bad Request", str(e.exception)) | |||
in_("start must precede end", str(e.exception)) | |||
# Check 404 for missing streams | |||
for function in [ client.stream_intervals, client.stream_extract ]: | |||
with assert_raises(ClientError) as e: | |||
@@ -208,12 +208,3 @@ class TestServer(object): | |||
data = getjson("/stream/get_metadata?path=/newton/prep" | |||
"&key=foo") | |||
eq_(data, {'foo': None}) | |||
def test_insert(self): | |||
# GET instead of POST (no body) | |||
# (actual POST test is done by client code) | |||
with assert_raises(HTTPError) as e: | |||
getjson("/stream/insert?path=/newton/prep&start=0&end=0") | |||
eq_(e.exception.code, 400) | |||