Browse Source

Fix line parsing in http client

tags/nilmdb-1.7.0
Jim Paris 11 years ago
parent
commit
dfdd0e5c74
1 changed files with 24 additions and 2 deletions
  1. +24
    -2
      nilmdb/client/httpclient.py

+ 24
- 2
nilmdb/client/httpclient.py View File

@@ -123,14 +123,36 @@ class HTTPClient(object):
"""
(response, isjson) = self._do_req(method, url, query, body,
stream = True, headers = headers)

# Like the iter_lines function in Requests, but only splits on
# the specified line ending.
def lines(source, ending):
pending = None
for chunk in source:
if pending is not None:
chunk = pending + chunk
tmp = chunk.split(ending)
lines = tmp[:-1]
if chunk.endswith(ending):
pending = None
else:
pending = tmp[-1]
for line in lines:
yield line
if pending is not None:
yield pending

# Yield the chunks or lines as requested
if binary:
for chunk in response.iter_content(chunk_size = 65536):
yield chunk
elif isjson:
for line in response.iter_lines():
for line in lines(response.iter_content(chunk_size = 1),
ending = '\r\n'):
yield json.loads(line)
else:
for line in response.iter_lines():
for line in lines(response.iter_content(chunk_size = 65536),
ending = '\n'):
yield line

def get_gen(self, url, params = None, binary = False):


Loading…
Cancel
Save