You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

208 lines
7.6 KiB

  1. """HTTP client library"""
  2. import nilmdb
  3. import nilmdb.utils
  4. from nilmdb.client.errors import ClientError, ServerError, Error
  5. import simplejson as json
  6. import urlparse
  7. import pycurl
  8. import cStringIO
  9. class HTTPClient(object):
  10. """Class to manage and perform HTTP requests from the client"""
  11. def __init__(self, baseurl = ""):
  12. """If baseurl is supplied, all other functions that take
  13. a URL can be given a relative URL instead."""
  14. # Verify / clean up URL
  15. reparsed = urlparse.urlparse(baseurl).geturl()
  16. if '://' not in reparsed:
  17. reparsed = urlparse.urlparse("http://" + baseurl).geturl()
  18. self.baseurl = reparsed
  19. self.curl = pycurl.Curl()
  20. self.curl.setopt(pycurl.SSL_VERIFYHOST, 2)
  21. self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
  22. self.curl.setopt(pycurl.MAXREDIRS, 5)
  23. self._setup_url()
  24. def _setup_url(self, url = "", params = ""):
  25. url = urlparse.urljoin(self.baseurl, url)
  26. if params:
  27. url = urlparse.urljoin(
  28. url, "?" + nilmdb.utils.urllib.urlencode(params))
  29. self.curl.setopt(pycurl.URL, url)
  30. self.url = url
  31. def _check_busy_and_set_upload(self, upload):
  32. """Sets the pycurl.UPLOAD option, but also raises a more
  33. friendly exception if the client is already serving a request."""
  34. try:
  35. self.curl.setopt(pycurl.UPLOAD, upload)
  36. except pycurl.error as e:
  37. if "is currently running" in str(e):
  38. raise Exception("Client is already performing a request, and "
  39. "nesting calls is not supported.")
  40. raise
  41. def _check_error(self, body = None):
  42. code = self.curl.getinfo(pycurl.RESPONSE_CODE)
  43. if code == 200:
  44. return
  45. # Default variables for exception. We use the entire body as
  46. # the default message, in case we can't extract it from a JSON
  47. # response.
  48. args = { "url" : self.url,
  49. "status" : str(code),
  50. "message" : body,
  51. "traceback" : None }
  52. try:
  53. # Fill with server-provided data if we can
  54. jsonerror = json.loads(body)
  55. args["status"] = jsonerror["status"]
  56. args["message"] = jsonerror["message"]
  57. args["traceback"] = jsonerror["traceback"]
  58. except Exception: # pragma: no cover
  59. pass
  60. if code >= 400 and code <= 499:
  61. raise ClientError(**args)
  62. else: # pragma: no cover
  63. if code >= 500 and code <= 599:
  64. if args["message"] is None:
  65. args["message"] = ("(no message; try disabling " +
  66. "response.stream option in " +
  67. "nilmdb.server for better debugging)")
  68. raise ServerError(**args)
  69. else:
  70. raise Error(**args)
  71. def _req_generator(self, url, params):
  72. """
  73. Like self._req(), but runs the perform in a separate thread.
  74. It returns a generator that spits out arbitrary-sized chunks
  75. of the resulting data, instead of using the WRITEFUNCTION
  76. callback.
  77. """
  78. self._setup_url(url, params)
  79. self._status = None
  80. error_body = ""
  81. self._headers = ""
  82. def header_callback(data):
  83. if self._status is None:
  84. self._status = int(data.split(" ")[1])
  85. self._headers += data
  86. self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
  87. def perform(callback):
  88. self.curl.setopt(pycurl.WRITEFUNCTION, callback)
  89. self.curl.perform()
  90. try:
  91. with nilmdb.utils.Iteratorizer(perform, curl_hack = True) as it:
  92. for i in it:
  93. if self._status == 200:
  94. # If we had a 200 response, yield the data to caller.
  95. yield i
  96. else:
  97. # Otherwise, collect it into an error string.
  98. error_body += i
  99. except pycurl.error as e:
  100. raise ServerError(status = "502 Error",
  101. url = self.url,
  102. message = e[1])
  103. # Raise an exception if there was an error
  104. self._check_error(error_body)
  105. def _req(self, url, params):
  106. """
  107. GET or POST that returns raw data. Returns the body
  108. data as a string, or raises an error if it contained an error.
  109. """
  110. self._setup_url(url, params)
  111. body = cStringIO.StringIO()
  112. self.curl.setopt(pycurl.WRITEFUNCTION, body.write)
  113. self._headers = ""
  114. def header_callback(data):
  115. self._headers += data
  116. self.curl.setopt(pycurl.HEADERFUNCTION, header_callback)
  117. try:
  118. self.curl.perform()
  119. except pycurl.error as e:
  120. raise ServerError(status = "502 Error",
  121. url = self.url,
  122. message = e[1])
  123. body_str = body.getvalue()
  124. # Raise an exception if there was an error
  125. self._check_error(body_str)
  126. return body_str
  127. def close(self):
  128. self.curl.close()
  129. def _iterate_lines(self, it):
  130. """
  131. Given an iterator that returns arbitrarily-sized chunks
  132. of data, return '\n'-delimited lines of text
  133. """
  134. partial = ""
  135. for chunk in it:
  136. partial += chunk
  137. lines = partial.split("\n")
  138. for line in lines[0:-1]:
  139. yield line
  140. partial = lines[-1]
  141. if partial != "":
  142. yield partial
  143. # Non-generator versions
  144. def _doreq(self, url, params, retjson):
  145. """
  146. Perform a request, and return the body.
  147. url: URL to request (relative to baseurl)
  148. params: dictionary of query parameters
  149. retjson: expect JSON and return python objects instead of string
  150. """
  151. out = self._req(url, params)
  152. if retjson:
  153. return json.loads(out)
  154. return out
  155. def get(self, url, params = None, retjson = True):
  156. """Simple GET"""
  157. self._check_busy_and_set_upload(0)
  158. return self._doreq(url, params, retjson)
  159. def put(self, url, postdata, params = None, retjson = True):
  160. """Simple PUT"""
  161. self._check_busy_and_set_upload(1)
  162. self._setup_url(url, params)
  163. data = cStringIO.StringIO(postdata)
  164. self.curl.setopt(pycurl.READFUNCTION, data.read)
  165. return self._doreq(url, params, retjson)
  166. # Generator versions
  167. def _doreq_gen(self, url, params, retjson):
  168. """
  169. Perform a request, and return lines of the body in a generator.
  170. url: URL to request (relative to baseurl)
  171. params: dictionary of query parameters
  172. retjson: expect JSON and yield python objects instead of strings
  173. """
  174. for line in self._iterate_lines(self._req_generator(url, params)):
  175. if retjson:
  176. yield json.loads(line)
  177. else:
  178. yield line
  179. def get_gen(self, url, params = None, retjson = True):
  180. """Simple GET, returning a generator"""
  181. self._check_busy_and_set_upload(0)
  182. return self._doreq_gen(url, params, retjson)
  183. def put_gen(self, url, postdata, params = None, retjson = True):
  184. """Simple PUT, returning a generator"""
  185. self._check_busy_and_set_upload(1)
  186. self._setup_url(url, params)
  187. data = cStringIO.StringIO(postdata)
  188. self.curl.setopt(pycurl.READFUNCTION, data.read)
  189. return self._doreq_gen(url, params, retjson)