You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

188 lines
7.6 KiB

  1. """HTTP client library"""
  2. import nilmdb.utils
  3. from nilmdb.client.errors import ClientError, ServerError, Error
  4. import simplejson as json
  5. import urllib.parse
  6. import requests
  7. class HTTPClient(object):
  8. """Class to manage and perform HTTP requests from the client"""
  9. def __init__(self, baseurl = "", post_json = False, verify_ssl = True):
  10. """If baseurl is supplied, all other functions that take
  11. a URL can be given a relative URL instead."""
  12. # Verify / clean up URL
  13. reparsed = urllib.parse.urlparse(baseurl).geturl()
  14. if '://' not in reparsed:
  15. reparsed = urllib.parse.urlparse("http://" + baseurl).geturl()
  16. self.baseurl = reparsed.rstrip('/') + '/'
  17. # Note whether we want SSL verification
  18. self.verify_ssl = verify_ssl
  19. # Saved response, so that tests can verify a few things.
  20. self._last_response = {}
  21. # Whether to send application/json POST bodies (versus
  22. # x-www-form-urlencoded)
  23. self.post_json = post_json
  24. def _handle_error(self, url, code, body):
  25. # Default variables for exception. We use the entire body as
  26. # the default message, in case we can't extract it from a JSON
  27. # response.
  28. args = { "url" : url,
  29. "status" : str(code),
  30. "message" : body,
  31. "traceback" : None }
  32. try:
  33. # Fill with server-provided data if we can
  34. jsonerror = json.loads(body)
  35. args["status"] = jsonerror["status"]
  36. args["message"] = jsonerror["message"]
  37. args["traceback"] = jsonerror["traceback"]
  38. except Exception:
  39. pass
  40. if code >= 400 and code <= 499:
  41. raise ClientError(**args)
  42. else:
  43. if code >= 500 and code <= 599:
  44. if args["message"] is None:
  45. args["message"] = ("(no message; try disabling " +
  46. "response.stream option in " +
  47. "nilmdb.server for better debugging)")
  48. raise ServerError(**args)
  49. else:
  50. raise Error(**args)
  51. def close(self):
  52. pass
  53. def _do_req(self, method, url, query_data, body_data, stream, headers):
  54. url = urllib.parse.urljoin(self.baseurl, url)
  55. try:
  56. # Create a new session, ensure we send "Connection: close",
  57. # and explicitly close connection after the transfer.
  58. # This is to avoid HTTP/1.1 persistent connections
  59. # (keepalive), because they have fundamental race
  60. # conditions when there are delays between requests:
  61. # a new request may be sent at the same instant that the
  62. # server decides to timeout the connection.
  63. session = requests.Session()
  64. if headers is None:
  65. headers = {}
  66. headers["Connection"] = "close"
  67. response = session.request(method, url,
  68. params = query_data,
  69. data = body_data,
  70. stream = stream,
  71. headers = headers,
  72. verify = self.verify_ssl)
  73. # Close the connection. If it's a generator (stream =
  74. # True), the requests library shouldn't actually close the
  75. # HTTP connection until all data has been read from the
  76. # response.
  77. session.close()
  78. except requests.RequestException as e:
  79. raise ServerError(status = "502 Error", url = url,
  80. message = str(e))
  81. if response.status_code != 200:
  82. self._handle_error(url, response.status_code, response.content)
  83. self._last_response = response
  84. if response.headers["content-type"] in ("application/json",
  85. "application/x-json-stream"):
  86. return (response, True)
  87. else:
  88. return (response, False)
  89. # Normal versions that return data directly
  90. def _req(self, method, url, query = None, body = None, headers = None):
  91. """
  92. Make a request and return the body data as a string or parsed
  93. JSON object, or raise an error if it contained an error.
  94. """
  95. (response, isjson) = self._do_req(method, url, query, body,
  96. stream = False, headers = headers)
  97. if isjson:
  98. return json.loads(response.content)
  99. return response.content
  100. def get(self, url, params = None):
  101. """Simple GET (parameters in URL)"""
  102. return self._req("GET", url, params, None)
  103. def post(self, url, params = None):
  104. """Simple POST (parameters in body)"""
  105. if self.post_json:
  106. return self._req("POST", url, None,
  107. json.dumps(params),
  108. { 'Content-type': 'application/json' })
  109. else:
  110. return self._req("POST", url, None, params)
  111. def put(self, url, data, params = None,
  112. content_type = "application/octet-stream"):
  113. """Simple PUT (parameters in URL, data in body)"""
  114. h = { 'Content-type': content_type }
  115. return self._req("PUT", url, query = params, body = data, headers = h)
  116. # Generator versions that return data one line at a time.
  117. def _req_gen(self, method, url, query = None, body = None,
  118. headers = None, binary = False):
  119. """
  120. Make a request and return a generator that gives back strings
  121. or JSON decoded lines of the body data, or raise an error if
  122. it contained an eror.
  123. """
  124. (response, isjson) = self._do_req(method, url, query, body,
  125. stream = True, headers = headers)
  126. # Like the iter_lines function in Requests, but only splits on
  127. # the specified line ending.
  128. def lines(source, ending):
  129. pending = None
  130. for chunk in source:
  131. if pending is not None:
  132. chunk = pending + chunk
  133. tmp = chunk.split(ending)
  134. lines = tmp[:-1]
  135. if chunk.endswith(ending):
  136. pending = None
  137. else:
  138. pending = tmp[-1]
  139. for line in lines:
  140. yield line
  141. if pending is not None:
  142. yield pending
  143. # Yield the chunks or lines as requested
  144. if binary:
  145. for chunk in response.iter_content(chunk_size = 65536):
  146. yield chunk
  147. elif isjson:
  148. for line in lines(response.iter_content(chunk_size = 1),
  149. ending = b'\r\n'):
  150. yield json.loads(line)
  151. else:
  152. for line in lines(response.iter_content(chunk_size = 65536),
  153. ending = b'\n'):
  154. yield line
  155. def get_gen(self, url, params = None, binary = False):
  156. """Simple GET (parameters in URL) returning a generator"""
  157. return self._req_gen("GET", url, params, binary = binary)
  158. def post_gen(self, url, params = None):
  159. """Simple POST (parameters in body) returning a generator"""
  160. if self.post_json:
  161. return self._req_gen("POST", url, None,
  162. json.dumps(params),
  163. { 'Content-type': 'application/json' })
  164. else:
  165. return self._req_gen("POST", url, None, params)
  166. # Not much use for a POST or PUT generator, since they don't
  167. # return much data.