Browse Source

Enforce method types, and require POST for actions that change things.

This is a pretty big change that will render existing clients unable
to modify the database, but it's important that we use POST or PUT
instead of GET for anything that may change state, in case this
is ever put behind a cache.
tags/nilmdb-1.2
Jim Paris 11 years ago
parent
commit
3d82888580
5 changed files with 67 additions and 50 deletions
  1. +5
    -5
      nilmdb/client/client.py
  2. +28
    -22
      nilmdb/client/httpclient.py
  3. +25
    -6
      nilmdb/server/server.py
  4. +9
    -8
      tests/test_client.py
  5. +0
    -9
      tests/test_nilmdb.py

+ 5
- 5
nilmdb/client/client.py View File

@@ -73,7 +73,7 @@ class Client(object):
"path": path,
"data": self._json_param(data)
}
return self.http.get("stream/set_metadata", params)
return self.http.post("stream/set_metadata", params)

def stream_update_metadata(self, path, data):
"""Update stream metadata from a dictionary"""
@@ -81,18 +81,18 @@ class Client(object):
"path": path,
"data": self._json_param(data)
}
return self.http.get("stream/update_metadata", params)
return self.http.post("stream/update_metadata", params)

def stream_create(self, path, layout):
"""Create a new stream"""
params = { "path": path,
"layout" : layout }
return self.http.get("stream/create", params)
return self.http.post("stream/create", params)

def stream_destroy(self, path):
"""Delete stream and its contents"""
params = { "path": path }
return self.http.get("stream/destroy", params)
return self.http.post("stream/destroy", params)

def stream_remove(self, path, start = None, end = None):
"""Remove data from the specified time range"""
@@ -103,7 +103,7 @@ class Client(object):
params["start"] = float_to_string(start)
if end is not None:
params["end"] = float_to_string(end)
return self.http.get("stream/remove", params)
return self.http.post("stream/remove", params)

@contextlib.contextmanager
def stream_insert_context(self, path, start = None, end = None):


+ 28
- 22
nilmdb/client/httpclient.py View File

@@ -40,11 +40,16 @@ class HTTPClient(object):
self.lock = HTTPClientLock()
self._setup_url()

def _setup_url(self, url = "", params = ""):
def _setup_url(self, url = "", params = "", post_params = False):
url = urlparse.urljoin(self.baseurl, url)
self.curl.setopt(pycurl.POST, 0)
if params:
url = urlparse.urljoin(
url, "?" + nilmdb.utils.urllib.urlencode(params))
param_string = nilmdb.utils.urllib.urlencode(params)
if post_params:
self.curl.setopt(pycurl.POSTFIELDS, param_string)
self.curl.setopt(pycurl.POST, 1)
else:
url = urlparse.urljoin(url, '?' + param_string)
self.curl.setopt(pycurl.URL, url)
self.url = url

@@ -79,12 +84,12 @@ class HTTPClient(object):
else:
raise Error(**args)

def _req_generator(self, url, params):
def _req_generator(self, url, params, post_params = False):
"""
Like self._req(), but returns a generator that spits out
arbitrary-sized chunks of the resulting data.
"""
self._setup_url(url, params)
self._setup_url(url, params, post_params)
error_body = ""
self._headers = ""
self._req_status = None
@@ -119,12 +124,12 @@ class HTTPClient(object):
# Raise an exception if there was an error
self._check_error(error_body)

def _req(self, url, params):
def _req(self, url, params, post_params = False):
"""
GET or POST that returns raw data. Returns the body
data as a string, or raises an error if it contained an error.
"""
self._setup_url(url, params)
self._setup_url(url, params, post_params)
body = cStringIO.StringIO()
self.curl.setopt(pycurl.WRITEFUNCTION, body.write)
self._headers = ""
@@ -170,21 +175,28 @@ class HTTPClient(object):

# Normal versions that return data directly
def get(self, url, params = None, retjson = True):
"""Simple GET"""
"""Simple GET (parameters in URL)"""
with self.lock:
self.curl.setopt(pycurl.UPLOAD, 0)
self.curl.setopt(pycurl.READFUNCTION, lambda: None)
return self._maybe_json(retjson, self._req(url, params, False))

def post(self, url, params = None, retjson = True):
"""Simple POST (parameters in body)"""
with self.lock:
self.curl.setopt(pycurl.UPLOAD, 0)
self.curl.setopt(pycurl.READFUNCTION, lambda: None)
return self._maybe_json(retjson, self._req(url, params))
return self._maybe_json(retjson, self._req(url, params, True))

def put(self, url, postdata, params = None, retjson = True):
"""Simple PUT"""
def put(self, url, data, params = None, retjson = True):
"""Simple PUT (parameters in URL, data in body)"""
with self.lock:
self.curl.setopt(pycurl.UPLOAD, 1)
data = cStringIO.StringIO(postdata)
data = cStringIO.StringIO(data)
self.curl.setopt(pycurl.READFUNCTION, data.read)
return self._maybe_json(retjson, self._req(url, params))
return self._maybe_json(retjson, self._req(url, params, False))

# Generator versions that return data one line at a time
# Generator versions that return data one line at a time.
def get_gen(self, url, params = None, retjson = True):
"""Simple GET that yields one resonse line at a time"""
with self.lock:
@@ -193,11 +205,5 @@ class HTTPClient(object):
for line in self._iterate_lines(self._req_generator(url, params)):
yield self._maybe_json(retjson, line)

def put_gen(self, url, postdata, params = None, retjson = True):
"""Simple PUT that yields one response line at a time"""
with self.lock:
self.curl.setopt(pycurl.UPLOAD, 1)
data = cStringIO.StringIO(postdata)
self.curl.setopt(pycurl.READFUNCTION, data.read)
for line in self._iterate_lines(self._req_generator(url, params)):
yield self._maybe_json(retjson, line)
# Not much use for a POST or PUT generator, since they don't
# return much data.

+ 25
- 6
nilmdb/server/server.py View File

@@ -71,6 +71,17 @@ def exception_to_httperror(*expected):
# care of that.
return decorator.decorator(wrapper)

# Custom Cherrypy tools
def allow_methods(methods):
method = cherrypy.request.method.upper()
if method not in methods:
if method in cherrypy.request.methods_with_bodies:
cherrypy.request.body.read()
allowed = ', '.join(methods)
cherrypy.response.headers['Allow'] = allowed
raise cherrypy.HTTPError(405, method + " not allowed; use " + allowed)
cherrypy.tools.allow_methods = cherrypy.Tool('before_handler', allow_methods)

# CherryPy apps
class Root(NilmApp):
"""Root application for NILM database"""
@@ -123,6 +134,7 @@ class Stream(NilmApp):
@cherrypy.expose
@cherrypy.tools.json_out()
@exception_to_httperror(NilmDBError, ValueError)
@cherrypy.tools.allow_methods(methods = ["POST"])
def create(self, path, layout):
"""Create a new stream in the database. Provide path
and one of the nilmdb.layout.layouts keys.
@@ -133,6 +145,7 @@ class Stream(NilmApp):
@cherrypy.expose
@cherrypy.tools.json_out()
@exception_to_httperror(NilmDBError)
@cherrypy.tools.allow_methods(methods = ["POST"])
def destroy(self, path):
"""Delete a stream and its associated data."""
return self.db.stream_destroy(path)
@@ -165,6 +178,7 @@ class Stream(NilmApp):
@cherrypy.expose
@cherrypy.tools.json_out()
@exception_to_httperror(NilmDBError, LookupError, TypeError)
@cherrypy.tools.allow_methods(methods = ["POST"])
def set_metadata(self, path, data):
"""Set metadata for the named stream, replacing any
existing metadata. Data should be a json-encoded
@@ -176,6 +190,7 @@ class Stream(NilmApp):
@cherrypy.expose
@cherrypy.tools.json_out()
@exception_to_httperror(NilmDBError, LookupError, TypeError)
@cherrypy.tools.allow_methods(methods = ["POST"])
def update_metadata(self, path, data):
"""Update metadata for the named stream. Data
should be a json-encoded dictionary"""
@@ -185,6 +200,7 @@ class Stream(NilmApp):
# /stream/insert?path=/newton/prep
@cherrypy.expose
@cherrypy.tools.json_out()
@cherrypy.tools.allow_methods(methods = ["PUT"])
def insert(self, path, start, end):
"""
Insert new data into the database. Provide textual data
@@ -192,12 +208,9 @@ class Stream(NilmApp):
"""
# Important that we always read the input before throwing any
# errors, to keep lengths happy for persistent connections.
# However, CherryPy 3.2.2 has a bug where this fails for GET
# requests, so catch that. (issue #1134)
try:
body = cherrypy.request.body.read()
except TypeError:
raise cherrypy.HTTPError("400 Bad Request", "No request body")
# Note that CherryPy 3.2.2 has a bug where this fails for GET
# requests, if we ever want to handle those (issue #1134)
body = cherrypy.request.body.read()

# Check path and get layout
streams = self.db.stream_list(path = path)
@@ -243,6 +256,7 @@ class Stream(NilmApp):
@cherrypy.expose
@cherrypy.tools.json_out()
@exception_to_httperror(NilmDBError)
@cherrypy.tools.allow_methods(methods = ["POST"])
def remove(self, path, start = None, end = None):
"""
Remove data from the backend database. Removes all data in
@@ -404,6 +418,11 @@ class Server(object):
app_config.update({ 'response.headers.Access-Control-Allow-Origin':
'*' })

# Only allow GET and HEAD by default. Individual handlers
# can override.
app_config.update({ 'tools.allow_methods.on': True,
'tools.allow_methods.methods': ['GET', 'HEAD'] })

# Send tracebacks in error responses. They're hidden by the
# error_page function for client errors (code 400-499).
app_config.update({ 'request.show_tracebacks' : True })


+ 9
- 8
tests/test_client.py View File

@@ -97,6 +97,15 @@ class TestClient(object):
with assert_raises(ClientError):
client.stream_create("/newton/prep", "NoSuchLayout")

# Bad method types
with assert_raises(ClientError):
client.http.put("/stream/list","")
# Try a bunch of times to make sure the request body is getting consumed
for x in range(10):
with assert_raises(ClientError):
client.http.post("/stream/list")
client = nilmdb.Client(url = testurl)

# Create three streams
client.stream_create("/newton/prep", "PrepData")
client.stream_create("/newton/raw", "RawData")
@@ -312,14 +321,6 @@ class TestClient(object):
for (a, b) in itertools.izip(aa, bb):
eq_(json.loads(a), b)

# Check PUT with generator out
with assert_raises(ClientError) as e:
client.http.put_gen("stream/insert", "",
{ "path": "/newton/prep",
"start": 0, "end": 0 }).next()
in_("400 Bad Request", str(e.exception))
in_("start must precede end", str(e.exception))

# Check 404 for missing streams
for function in [ client.stream_intervals, client.stream_extract ]:
with assert_raises(ClientError) as e:


+ 0
- 9
tests/test_nilmdb.py View File

@@ -208,12 +208,3 @@ class TestServer(object):
data = getjson("/stream/get_metadata?path=/newton/prep"
"&key=foo")
eq_(data, {'foo': None})


def test_insert(self):
# GET instead of POST (no body)
# (actual POST test is done by client code)
with assert_raises(HTTPError) as e:
getjson("/stream/insert?path=/newton/prep&start=0&end=0")
eq_(e.exception.code, 400)


Loading…
Cancel
Save