|
|
@@ -123,14 +123,36 @@ class HTTPClient(object): |
|
|
|
""" |
|
|
|
(response, isjson) = self._do_req(method, url, query, body, |
|
|
|
stream = True, headers = headers) |
|
|
|
|
|
|
|
# Like the iter_lines function in Requests, but only splits on |
|
|
|
# the specified line ending. |
|
|
|
def lines(source, ending): |
|
|
|
pending = None |
|
|
|
for chunk in source: |
|
|
|
if pending is not None: |
|
|
|
chunk = pending + chunk |
|
|
|
tmp = chunk.split(ending) |
|
|
|
lines = tmp[:-1] |
|
|
|
if chunk.endswith(ending): |
|
|
|
pending = None |
|
|
|
else: |
|
|
|
pending = tmp[-1] |
|
|
|
for line in lines: |
|
|
|
yield line |
|
|
|
if pending is not None: |
|
|
|
yield pending |
|
|
|
|
|
|
|
# Yield the chunks or lines as requested |
|
|
|
if binary: |
|
|
|
for chunk in response.iter_content(chunk_size = 65536): |
|
|
|
yield chunk |
|
|
|
elif isjson: |
|
|
|
for line in response.iter_lines(): |
|
|
|
for line in lines(response.iter_content(chunk_size = 1), |
|
|
|
ending = '\r\n'): |
|
|
|
yield json.loads(line) |
|
|
|
else: |
|
|
|
for line in response.iter_lines(): |
|
|
|
for line in lines(response.iter_content(chunk_size = 65536), |
|
|
|
ending = '\n'): |
|
|
|
yield line |
|
|
|
|
|
|
|
def get_gen(self, url, params = None, binary = False): |
|
|
|