|
|
@@ -6,21 +6,7 @@ from nilmdb.client.errors import ClientError, ServerError, Error |
|
|
|
|
|
|
|
import simplejson as json |
|
|
|
import urlparse |
|
|
|
import pycurl |
|
|
|
import cStringIO |
|
|
|
import threading |
|
|
|
|
|
|
|
import warnings # can remove |
|
|
|
|
|
|
|
class HTTPClientLock(object): |
|
|
|
def __init__(self): |
|
|
|
self.lock = threading.Lock() |
|
|
|
def __enter__(self): |
|
|
|
if not self.lock.acquire(False): |
|
|
|
raise Exception("Client is already performing a request, and " |
|
|
|
"nested or concurrent calls are not supported.") |
|
|
|
def __exit__(self, exc_type, exc_value, traceback): |
|
|
|
self.lock.release() |
|
|
|
import requests |
|
|
|
|
|
|
|
class HTTPClient(object): |
|
|
|
"""Class to manage and perform HTTP requests from the client""" |
|
|
@@ -32,42 +18,19 @@ class HTTPClient(object): |
|
|
|
if '://' not in reparsed: |
|
|
|
reparsed = urlparse.urlparse("http://" + baseurl).geturl() |
|
|
|
self.baseurl = reparsed |
|
|
|
self.multi = pycurl.CurlMulti() |
|
|
|
self.curl = pycurl.Curl() |
|
|
|
# Add and remove the handle to workaround a curl bug (debian #701713) |
|
|
|
self.multi.add_handle(self.curl) |
|
|
|
self.multi.remove_handle(self.curl) |
|
|
|
#self.multi = nilmdb.utils.threadsafety.verify_proxy(pycurl.CurlMulti)() |
|
|
|
#self.curl = nilmdb.utils.threadsafety.verify_proxy(pycurl.Curl)() |
|
|
|
self.curl.setopt(pycurl.SSL_VERIFYHOST, 2) |
|
|
|
self.curl.setopt(pycurl.FOLLOWLOCATION, 1) |
|
|
|
self.curl.setopt(pycurl.MAXREDIRS, 5) |
|
|
|
self.curl.setopt(pycurl.NOSIGNAL, 1) |
|
|
|
self.lock = HTTPClientLock() |
|
|
|
self._setup_url() |
|
|
|
|
|
|
|
def _setup_url(self, url = "", params = None, post = False): |
|
|
|
url = urlparse.urljoin(self.baseurl, url) |
|
|
|
if params is None: |
|
|
|
encoded = "" |
|
|
|
else: |
|
|
|
encoded = nilmdb.utils.urllib.urlencode(params) |
|
|
|
self.curl.setopt(pycurl.POST, 1 if post else 0) |
|
|
|
if post: |
|
|
|
self.curl.setopt(pycurl.POSTFIELDS, encoded) |
|
|
|
else: |
|
|
|
if encoded: |
|
|
|
url = urlparse.urljoin(url, '?' + encoded) |
|
|
|
self.curl.setopt(pycurl.URL, url) |
|
|
|
self.url = url |
|
|
|
# Build Requests session object, enable SSL verification |
|
|
|
self.session = requests.Session() |
|
|
|
self.session.verify = True |
|
|
|
|
|
|
|
# Saved headers, for tests to check |
|
|
|
self._headers = {} |
|
|
|
|
|
|
|
def _check_error(self, code, body): |
|
|
|
def _handle_error(self, url, code, body): |
|
|
|
# Default variables for exception. We use the entire body as |
|
|
|
# the default message, in case we can't extract it from a JSON |
|
|
|
# response. |
|
|
|
if code == 200: |
|
|
|
return |
|
|
|
args = { "url" : self.url, |
|
|
|
args = { "url" : url, |
|
|
|
"status" : str(code), |
|
|
|
"message" : body, |
|
|
|
"traceback" : None } |
|
|
@@ -91,137 +54,67 @@ class HTTPClient(object): |
|
|
|
else: |
|
|
|
raise Error(**args) |
|
|
|
|
|
|
|
def _req_generator(self, url, params, post_params = False): |
|
|
|
""" |
|
|
|
Like self._req(), but returns a generator that spits out |
|
|
|
arbitrary-sized chunks of the resulting data. |
|
|
|
""" |
|
|
|
self._setup_url(url, params, post_params) |
|
|
|
data = [] |
|
|
|
error_body = [] |
|
|
|
self.headers = cStringIO.StringIO() |
|
|
|
req_status = [None] |
|
|
|
req_expect_status = [True] |
|
|
|
def header_callback(data): |
|
|
|
"""Process header data as it comes in, so we can strip out |
|
|
|
the status line.""" |
|
|
|
# This weirdness is to handle the fact that |
|
|
|
# there can be a "HTTP/1.1 100 Continue" block before |
|
|
|
# the real response. Save all statuses that we see. |
|
|
|
if req_expect_status[0]: |
|
|
|
req_status[0] = int(data.split(" ")[1]) |
|
|
|
req_expect_status[0] = False |
|
|
|
elif data == "\r\n": |
|
|
|
req_expect_status[0] = True |
|
|
|
self.headers.write(data) |
|
|
|
self.curl.setopt(pycurl.HEADERFUNCTION, header_callback) |
|
|
|
def close(self): |
|
|
|
self.session.close() |
|
|
|
|
|
|
|
# Use the Multi object so we don't have to block in a callback |
|
|
|
def write_callback(newdata): |
|
|
|
data.append(newdata) |
|
|
|
self.curl.setopt(pycurl.WRITEFUNCTION, write_callback) |
|
|
|
self.multi.add_handle(self.curl) |
|
|
|
def _do_req(self, method, url, query_data, body_data, stream): |
|
|
|
url = urlparse.urljoin(self.baseurl, url) |
|
|
|
try: |
|
|
|
while True: |
|
|
|
self.multi.select() |
|
|
|
with nilmdb.utils.Timer("perform"): |
|
|
|
(ret, handles) = self.multi.perform() |
|
|
|
# Check data first |
|
|
|
if len(data): |
|
|
|
if req_status[0] == 200: |
|
|
|
# If we had a 200 response, yield the data to caller. |
|
|
|
yield "".join(data) |
|
|
|
else: |
|
|
|
# Otherwise, collect it into an error string. |
|
|
|
error_body.extend(data) |
|
|
|
data = [] |
|
|
|
# If we got data, we're doing well; call perform again |
|
|
|
continue |
|
|
|
(in_queue, ok_objects, error_objects) = self.multi.info_read() |
|
|
|
if error_objects: |
|
|
|
raise ServerError(status = "502 Error", |
|
|
|
url = self.url, |
|
|
|
message = error_objects[0][2]) |
|
|
|
if ret == pycurl.E_CALL_MULTI_PERFORM: |
|
|
|
continue |
|
|
|
if handles == 0: |
|
|
|
break |
|
|
|
finally: |
|
|
|
# Have to pull out info before removing handle |
|
|
|
self._num_connects = self.curl.getinfo(pycurl.NUM_CONNECTS) |
|
|
|
code = self.curl.getinfo(pycurl.RESPONSE_CODE) |
|
|
|
# Remove handle |
|
|
|
self.multi.remove_handle(self.curl) |
|
|
|
self._check_error(code, "".join(error_body)) |
|
|
|
|
|
|
|
def _req(self, url, params, post_params = False): |
|
|
|
""" |
|
|
|
GET or POST that returns raw data. Returns the body |
|
|
|
data as a string, or raises an error if it contained an error. |
|
|
|
""" |
|
|
|
print "_req", url, params, post_params |
|
|
|
body = [] |
|
|
|
for data in self._req_generator(url, params, post_params): |
|
|
|
body.append(data) |
|
|
|
return "".join(body) |
|
|
|
|
|
|
|
def close(self): |
|
|
|
with self.lock: |
|
|
|
self.curl.close() |
|
|
|
response = self.session.request(method, url, |
|
|
|
params = query_data, |
|
|
|
data = body_data) |
|
|
|
except requests.RequestException as e: |
|
|
|
raise ServerError(status = "502 Error", url = url, |
|
|
|
message = str(e.message)) |
|
|
|
if response.status_code != 200: |
|
|
|
self._handle_error(url, response.status_code, response.content) |
|
|
|
self._headers = response.headers |
|
|
|
if response.headers["content-type"] in ("application/json", |
|
|
|
"application/x-json-stream"): |
|
|
|
return (response, True) |
|
|
|
else: |
|
|
|
return (response, False) |
|
|
|
|
|
|
|
def _iterate_lines(self, it): |
|
|
|
# Normal versions that return data directly |
|
|
|
def _req(self, method, url, query = None, body = None): |
|
|
|
""" |
|
|
|
Given an iterator that returns arbitrarily-sized chunks |
|
|
|
of data, return '\n'-delimited lines of text |
|
|
|
Make a request and return the body data as a string or parsed |
|
|
|
JSON object, or raise an error if it contained an error. |
|
|
|
""" |
|
|
|
partial = "" |
|
|
|
for chunk in it: |
|
|
|
partial += chunk |
|
|
|
lines = partial.split("\n") |
|
|
|
for line in lines[0:-1]: |
|
|
|
yield line |
|
|
|
partial = lines[-1] |
|
|
|
if partial != "": |
|
|
|
yield partial |
|
|
|
|
|
|
|
def _maybe_json(self, retjson, str): |
|
|
|
"""Parse str as JSON if retjson is true, otherwise return |
|
|
|
it directly.""" |
|
|
|
if retjson: |
|
|
|
return json.loads(str) |
|
|
|
return str |
|
|
|
(response, isjson) = self._do_req(method, url, query, body, False) |
|
|
|
if isjson: |
|
|
|
return json.loads(response.content) |
|
|
|
return response.content |
|
|
|
|
|
|
|
# Normal versions that return data directly |
|
|
|
def get(self, url, params = None, retjson = True): |
|
|
|
def get(self, url, params = None): |
|
|
|
"""Simple GET (parameters in URL)""" |
|
|
|
with self.lock: |
|
|
|
self.curl.setopt(pycurl.UPLOAD, 0) |
|
|
|
self.curl.setopt(pycurl.READFUNCTION, lambda: None) |
|
|
|
return self._maybe_json(retjson, self._req(url, params, False)) |
|
|
|
return self._req("GET", url, params, None) |
|
|
|
|
|
|
|
def post(self, url, params = None, retjson = True): |
|
|
|
def post(self, url, params = None): |
|
|
|
"""Simple POST (parameters in body)""" |
|
|
|
with self.lock: |
|
|
|
self.curl.setopt(pycurl.UPLOAD, 0) |
|
|
|
self.curl.setopt(pycurl.READFUNCTION, lambda: None) |
|
|
|
return self._maybe_json(retjson, self._req(url, params, True)) |
|
|
|
return self._req("POST", url, None, params) |
|
|
|
|
|
|
|
def put(self, url, data, params = None, retjson = True): |
|
|
|
def put(self, url, data, params = None): |
|
|
|
"""Simple PUT (parameters in URL, data in body)""" |
|
|
|
with self.lock: |
|
|
|
self.curl.setopt(pycurl.UPLOAD, 1) |
|
|
|
data = cStringIO.StringIO(data) |
|
|
|
self.curl.setopt(pycurl.READFUNCTION, data.read) |
|
|
|
return self._maybe_json(retjson, self._req(url, params, False)) |
|
|
|
return self._req("PUT", url, params, data) |
|
|
|
|
|
|
|
# Generator versions that return data one line at a time. |
|
|
|
def get_gen(self, url, params = None, retjson = True): |
|
|
|
"""Simple GET that yields one resonse line at a time""" |
|
|
|
with self.lock: |
|
|
|
self.curl.setopt(pycurl.UPLOAD, 0) |
|
|
|
self.curl.setopt(pycurl.READFUNCTION, lambda: None) |
|
|
|
for line in self._iterate_lines(self._req_generator(url, params)): |
|
|
|
yield self._maybe_json(retjson, line) |
|
|
|
def _req_gen(self, method, url, query = None, body = None): |
|
|
|
""" |
|
|
|
Make a request and return a generator that gives back strings |
|
|
|
or JSON decoded lines of the body data, or raise an error if |
|
|
|
it contained an eror. |
|
|
|
""" |
|
|
|
(response, isjson) = self._do_req(method, url, query, body, True) |
|
|
|
for line in response.iter_lines(): |
|
|
|
if isjson: |
|
|
|
yield json.loads(line) |
|
|
|
else: |
|
|
|
yield line |
|
|
|
|
|
|
|
def get_gen(self, url, params = None): |
|
|
|
"""Simple GET (parameters in URL) returning a generator""" |
|
|
|
return self._req_gen("GET", url, params) |
|
|
|
|
|
|
|
# Not much use for a POST or PUT generator, since they don't |
|
|
|
# return much data. |