@@ -21,7 +21,8 @@ docs: | |||
make -C docs | |||
lint: | |||
pylint3 --rcfile=.pylintrc nilmdb | |||
flake8 nilmdb --exclude=fsck.py,nilmdb_fsck.py | |||
ctrl: lint | |||
test: | |||
ifneq ($(INSIDE_EMACS),) | |||
@@ -1,14 +1,5 @@ | |||
"""Main NilmDB import""" | |||
# These aren't imported automatically, because loading the server | |||
# stuff isn't always necessary. | |||
#from nilmdb.server import NilmDB, Server | |||
#from nilmdb.client import Client | |||
from nilmdb._version import get_versions | |||
__version__ = get_versions()['version'] | |||
del get_versions | |||
from ._version import get_versions | |||
__version__ = get_versions()['version'] | |||
del get_versions |
@@ -6,20 +6,21 @@ import nilmdb.utils | |||
import nilmdb.client.httpclient | |||
from nilmdb.client.errors import ClientError | |||
import time | |||
import json | |||
import contextlib | |||
from nilmdb.utils.time import timestamp_to_string, string_to_timestamp | |||
def extract_timestamp(line): | |||
"""Extract just the timestamp from a line of data text""" | |||
return string_to_timestamp(line.split()[0]) | |||
class Client(object): | |||
"""Main client interface to the Nilm database.""" | |||
def __init__(self, url, post_json = False): | |||
def __init__(self, url, post_json=False): | |||
"""Initialize client with given URL. If post_json is true, | |||
POST requests are sent with Content-Type 'application/json' | |||
instead of the default 'x-www-form-urlencoded'.""" | |||
@@ -38,7 +39,7 @@ class Client(object): | |||
if self.post_json: | |||
# If we're posting as JSON, we don't need to encode it further here | |||
return data | |||
return json.dumps(data, separators=(',',':')) | |||
return json.dumps(data, separators=(',', ':')) | |||
def close(self): | |||
"""Close the connection; safe to call multiple times""" | |||
@@ -57,7 +58,7 @@ class Client(object): | |||
as a dictionary.""" | |||
return self.http.get("dbinfo") | |||
def stream_list(self, path = None, layout = None, extended = False): | |||
def stream_list(self, path=None, layout=None, extended=False): | |||
"""Return a sorted list of [path, layout] lists. If 'path' or | |||
'layout' are specified, only return streams that match those | |||
exact values. If 'extended' is True, the returned lists have | |||
@@ -71,11 +72,11 @@ class Client(object): | |||
if extended: | |||
params["extended"] = 1 | |||
streams = self.http.get("stream/list", params) | |||
return nilmdb.utils.sort.sort_human(streams, key = lambda s: s[0]) | |||
return nilmdb.utils.sort.sort_human(streams, key=lambda s: s[0]) | |||
def stream_get_metadata(self, path, keys = None): | |||
def stream_get_metadata(self, path, keys=None): | |||
"""Get stream metadata""" | |||
params = { "path": path } | |||
params = {"path": path} | |||
if keys is not None: | |||
params["key"] = keys | |||
return self.http.get("stream/get_metadata", params) | |||
@@ -86,7 +87,7 @@ class Client(object): | |||
params = { | |||
"path": path, | |||
"data": self._json_post_param(data) | |||
} | |||
} | |||
return self.http.post("stream/set_metadata", params) | |||
def stream_update_metadata(self, path, data): | |||
@@ -94,27 +95,33 @@ class Client(object): | |||
params = { | |||
"path": path, | |||
"data": self._json_post_param(data) | |||
} | |||
} | |||
return self.http.post("stream/update_metadata", params) | |||
def stream_create(self, path, layout): | |||
"""Create a new stream""" | |||
params = { "path": path, | |||
"layout" : layout } | |||
params = { | |||
"path": path, | |||
"layout": layout | |||
} | |||
return self.http.post("stream/create", params) | |||
def stream_destroy(self, path): | |||
"""Delete stream. Fails if any data is still present.""" | |||
params = { "path": path } | |||
params = { | |||
"path": path | |||
} | |||
return self.http.post("stream/destroy", params) | |||
def stream_rename(self, oldpath, newpath): | |||
"""Rename a stream.""" | |||
params = { "oldpath": oldpath, | |||
"newpath": newpath } | |||
params = { | |||
"oldpath": oldpath, | |||
"newpath": newpath | |||
} | |||
return self.http.post("stream/rename", params) | |||
def stream_remove(self, path, start = None, end = None): | |||
def stream_remove(self, path, start=None, end=None): | |||
"""Remove data from the specified time range""" | |||
params = { | |||
"path": path | |||
@@ -129,7 +136,7 @@ class Client(object): | |||
return total | |||
@contextlib.contextmanager | |||
def stream_insert_context(self, path, start = None, end = None): | |||
def stream_insert_context(self, path, start=None, end=None): | |||
"""Return a context manager that allows data to be efficiently | |||
inserted into a stream in a piecewise manner. Data is | |||
provided as ASCII lines, and is aggregated and sent to the | |||
@@ -152,7 +159,7 @@ class Client(object): | |||
ctx.finalize() | |||
ctx.destroy() | |||
def stream_insert(self, path, data, start = None, end = None): | |||
def stream_insert(self, path, data, start=None, end=None): | |||
"""Insert rows of data into a stream. data should be a string | |||
or iterable that provides ASCII data that matches the database | |||
layout for path. Data is passed through stream_insert_context, | |||
@@ -166,7 +173,7 @@ class Client(object): | |||
ctx.insert(chunk) | |||
return ctx.last_response | |||
def stream_insert_block(self, path, data, start, end, binary = False): | |||
def stream_insert_block(self, path, data, start, end, binary=False): | |||
"""Insert a single fixed block of data into the stream. It is | |||
sent directly to the server in one block with no further | |||
processing. | |||
@@ -183,7 +190,7 @@ class Client(object): | |||
params["binary"] = 1 | |||
return self.http.put("stream/insert", data, params) | |||
def stream_intervals(self, path, start = None, end = None, diffpath = None): | |||
def stream_intervals(self, path, start=None, end=None, diffpath=None): | |||
""" | |||
Return a generator that yields each stream interval. | |||
@@ -201,8 +208,8 @@ class Client(object): | |||
params["end"] = timestamp_to_string(end) | |||
return self.http.get_gen("stream/intervals", params) | |||
def stream_extract(self, path, start = None, end = None, | |||
count = False, markup = False, binary = False): | |||
def stream_extract(self, path, start=None, end=None, | |||
count=False, markup=False, binary=False): | |||
""" | |||
Extract data from a stream. Returns a generator that yields | |||
lines of ASCII-formatted data that matches the database | |||
@@ -232,16 +239,17 @@ class Client(object): | |||
params["markup"] = 1 | |||
if binary: | |||
params["binary"] = 1 | |||
return self.http.get_gen("stream/extract", params, binary = binary) | |||
return self.http.get_gen("stream/extract", params, binary=binary) | |||
def stream_count(self, path, start = None, end = None): | |||
def stream_count(self, path, start=None, end=None): | |||
""" | |||
Return the number of rows of data in the stream that satisfy | |||
the given timestamps. | |||
""" | |||
counts = list(self.stream_extract(path, start, end, count = True)) | |||
counts = list(self.stream_extract(path, start, end, count=True)) | |||
return int(counts[0]) | |||
class StreamInserter(object): | |||
"""Object returned by stream_insert_context() that manages | |||
the insertion of rows of data into a particular path. | |||
@@ -330,7 +338,7 @@ class StreamInserter(object): | |||
# Send the block once we have enough data | |||
if self._block_len >= maxdata: | |||
self._send_block(final = False) | |||
self._send_block(final=False) | |||
if self._block_len >= self._max_data_after_send: | |||
raise ValueError("too much data left over after trying" | |||
" to send intermediate block; is it" | |||
@@ -357,12 +365,12 @@ class StreamInserter(object): | |||
If more data is inserted after a finalize(), it will become | |||
part of a new interval and there may be a gap left in-between.""" | |||
self._send_block(final = True) | |||
self._send_block(final=True) | |||
def send(self): | |||
"""Send any data that we might have buffered up. Does not affect | |||
any other treatment of timestamps or endpoints.""" | |||
self._send_block(final = False) | |||
self._send_block(final=False) | |||
def _get_first_noncomment(self, block): | |||
"""Return the (start, end) indices of the first full line in | |||
@@ -392,7 +400,7 @@ class StreamInserter(object): | |||
raise IndexError | |||
end = start | |||
def _send_block(self, final = False): | |||
def _send_block(self, final=False): | |||
"""Send data currently in the block. The data sent will | |||
consist of full lines only, so some might be left over.""" | |||
# Build the full string to send | |||
@@ -405,7 +413,7 @@ class StreamInserter(object): | |||
(spos, epos) = self._get_first_noncomment(block) | |||
start_ts = extract_timestamp(block[spos:epos]) | |||
except (ValueError, IndexError): | |||
pass # no timestamp is OK, if we have no data | |||
pass # no timestamp is OK, if we have no data | |||
if final: | |||
# For a final block, it must end in a newline, and the | |||
@@ -420,7 +428,7 @@ class StreamInserter(object): | |||
end_ts = extract_timestamp(block[spos:epos]) | |||
end_ts += nilmdb.utils.time.epsilon | |||
except (ValueError, IndexError): | |||
pass # no timestamp is OK, if we have no data | |||
pass # no timestamp is OK, if we have no data | |||
self._block_data = [] | |||
self._block_len = 0 | |||
@@ -447,7 +455,7 @@ class StreamInserter(object): | |||
# the server complain so that the error is the same | |||
# as if we hadn't done this chunking. | |||
end_ts = self._interval_end | |||
self._block_data = [ block[spos:] ] | |||
self._block_data = [block[spos:]] | |||
self._block_len = (epos - spos) | |||
block = block[:spos] | |||
@@ -465,6 +473,6 @@ class StreamInserter(object): | |||
# Send it | |||
self.last_response = self._client.stream_insert_block( | |||
self._path, block, start_ts, end_ts, binary = False) | |||
self._path, block, start_ts, end_ts, binary=False) | |||
return |
@@ -1,19 +1,21 @@ | |||
"""HTTP client errors""" | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import sprintf | |||
class Error(Exception): | |||
"""Base exception for both ClientError and ServerError responses""" | |||
def __init__(self, | |||
status = "Unspecified error", | |||
message = None, | |||
url = None, | |||
traceback = None): | |||
status="Unspecified error", | |||
message=None, | |||
url=None, | |||
traceback=None): | |||
super().__init__(status) | |||
self.status = status # e.g. "400 Bad Request" | |||
self.message = message # textual message from the server | |||
self.url = url # URL we were requesting | |||
self.traceback = traceback # server traceback, if available | |||
self.traceback = traceback # server traceback, if available | |||
def _format_error(self, show_url): | |||
s = sprintf("[%s]", self.status) | |||
if self.message: | |||
@@ -23,11 +25,17 @@ class Error(Exception): | |||
if self.traceback: | |||
s += sprintf("\nServer traceback:\n%s", self.traceback) | |||
return s | |||
def __str__(self): | |||
return self._format_error(show_url = False) | |||
return self._format_error(show_url=False) | |||
def __repr__(self): | |||
return self._format_error(show_url = True) | |||
return self._format_error(show_url=True) | |||
class ClientError(Error): | |||
pass | |||
class ServerError(Error): | |||
pass |
@@ -1,15 +1,15 @@ | |||
"""HTTP client library""" | |||
import nilmdb.utils | |||
from nilmdb.client.errors import ClientError, ServerError, Error | |||
import json | |||
import urllib.parse | |||
import requests | |||
class HTTPClient(object): | |||
"""Class to manage and perform HTTP requests from the client""" | |||
def __init__(self, baseurl = "", post_json = False, verify_ssl = True): | |||
def __init__(self, baseurl="", post_json=False, verify_ssl=True): | |||
"""If baseurl is supplied, all other functions that take | |||
a URL can be given a relative URL instead.""" | |||
# Verify / clean up URL | |||
@@ -32,10 +32,12 @@ class HTTPClient(object): | |||
# Default variables for exception. We use the entire body as | |||
# the default message, in case we can't extract it from a JSON | |||
# response. | |||
args = { "url" : url, | |||
"status" : str(code), | |||
"message" : body, | |||
"traceback" : None } | |||
args = { | |||
"url": url, | |||
"status": str(code), | |||
"message": body, | |||
"traceback": None | |||
} | |||
try: | |||
# Fill with server-provided data if we can | |||
jsonerror = json.loads(body) | |||
@@ -49,8 +51,8 @@ class HTTPClient(object): | |||
else: | |||
if code >= 500 and code <= 599: | |||
if args["message"] is None: | |||
args["message"] = ("(no message; try disabling " + | |||
"response.stream option in " + | |||
args["message"] = ("(no message; try disabling " | |||
"response.stream option in " | |||
"nilmdb.server for better debugging)") | |||
raise ServerError(**args) | |||
else: | |||
@@ -74,11 +76,11 @@ class HTTPClient(object): | |||
headers = {} | |||
headers["Connection"] = "close" | |||
response = session.request(method, url, | |||
params = query_data, | |||
data = body_data, | |||
stream = stream, | |||
headers = headers, | |||
verify = self.verify_ssl) | |||
params=query_data, | |||
data=body_data, | |||
stream=stream, | |||
headers=headers, | |||
verify=self.verify_ssl) | |||
# Close the connection. If it's a generator (stream = | |||
# True), the requests library shouldn't actually close the | |||
@@ -86,8 +88,8 @@ class HTTPClient(object): | |||
# response. | |||
session.close() | |||
except requests.RequestException as e: | |||
raise ServerError(status = "502 Error", url = url, | |||
message = str(e)) | |||
raise ServerError(status="502 Error", url=url, | |||
message=str(e)) | |||
if response.status_code != 200: | |||
self._handle_error(url, response.status_code, response.content) | |||
self._last_response = response | |||
@@ -98,46 +100,46 @@ class HTTPClient(object): | |||
return (response, False) | |||
# Normal versions that return data directly | |||
def _req(self, method, url, query = None, body = None, headers = None): | |||
def _req(self, method, url, query=None, body=None, headers=None): | |||
""" | |||
Make a request and return the body data as a string or parsed | |||
JSON object, or raise an error if it contained an error. | |||
""" | |||
(response, isjson) = self._do_req(method, url, query, body, | |||
stream = False, headers = headers) | |||
stream=False, headers=headers) | |||
if isjson: | |||
return json.loads(response.content) | |||
return response.content | |||
def get(self, url, params = None): | |||
def get(self, url, params=None): | |||
"""Simple GET (parameters in URL)""" | |||
return self._req("GET", url, params, None) | |||
def post(self, url, params = None): | |||
def post(self, url, params=None): | |||
"""Simple POST (parameters in body)""" | |||
if self.post_json: | |||
return self._req("POST", url, None, | |||
json.dumps(params), | |||
{ 'Content-type': 'application/json' }) | |||
{'Content-type': 'application/json'}) | |||
else: | |||
return self._req("POST", url, None, params) | |||
def put(self, url, data, params = None, | |||
content_type = "application/octet-stream"): | |||
def put(self, url, data, params=None, | |||
content_type="application/octet-stream"): | |||
"""Simple PUT (parameters in URL, data in body)""" | |||
h = { 'Content-type': content_type } | |||
return self._req("PUT", url, query = params, body = data, headers = h) | |||
h = {'Content-type': content_type} | |||
return self._req("PUT", url, query=params, body=data, headers=h) | |||
# Generator versions that return data one line at a time. | |||
def _req_gen(self, method, url, query = None, body = None, | |||
headers = None, binary = False): | |||
def _req_gen(self, method, url, query=None, body=None, | |||
headers=None, binary=False): | |||
""" | |||
Make a request and return a generator that gives back strings | |||
or JSON decoded lines of the body data, or raise an error if | |||
it contained an eror. | |||
""" | |||
(response, isjson) = self._do_req(method, url, query, body, | |||
stream = True, headers = headers) | |||
stream=True, headers=headers) | |||
# Like the iter_lines function in Requests, but only splits on | |||
# the specified line ending. | |||
@@ -159,27 +161,27 @@ class HTTPClient(object): | |||
# Yield the chunks or lines as requested | |||
if binary: | |||
for chunk in response.iter_content(chunk_size = 65536): | |||
for chunk in response.iter_content(chunk_size=65536): | |||
yield chunk | |||
elif isjson: | |||
for line in lines(response.iter_content(chunk_size = 1), | |||
ending = b'\r\n'): | |||
for line in lines(response.iter_content(chunk_size=1), | |||
ending=b'\r\n'): | |||
yield json.loads(line) | |||
else: | |||
for line in lines(response.iter_content(chunk_size = 65536), | |||
ending = b'\n'): | |||
for line in lines(response.iter_content(chunk_size=65536), | |||
ending=b'\n'): | |||
yield line | |||
def get_gen(self, url, params = None, binary = False): | |||
def get_gen(self, url, params=None, binary=False): | |||
"""Simple GET (parameters in URL) returning a generator""" | |||
return self._req_gen("GET", url, params, binary = binary) | |||
return self._req_gen("GET", url, params, binary=binary) | |||
def post_gen(self, url, params = None): | |||
def post_gen(self, url, params=None): | |||
"""Simple POST (parameters in body) returning a generator""" | |||
if self.post_json: | |||
return self._req_gen("POST", url, None, | |||
json.dumps(params), | |||
{ 'Content-type': 'application/json' }) | |||
{'Content-type': 'application/json'}) | |||
else: | |||
return self._req_gen("POST", url, None, params) | |||
@@ -9,10 +9,9 @@ import nilmdb.client.httpclient | |||
from nilmdb.client.errors import ClientError | |||
import contextlib | |||
from nilmdb.utils.time import timestamp_to_string, string_to_timestamp | |||
import numpy | |||
import io | |||
def layout_to_dtype(layout): | |||
ltype = layout.split('_')[0] | |||
@@ -31,6 +30,7 @@ def layout_to_dtype(layout): | |||
dtype = [('timestamp', '<i8'), ('data', atype, lcount)] | |||
return numpy.dtype(dtype) | |||
class NumpyClient(nilmdb.client.client.Client): | |||
"""Subclass of nilmdb.client.Client that adds additional methods for | |||
extracting and inserting data via Numpy arrays.""" | |||
@@ -43,9 +43,9 @@ class NumpyClient(nilmdb.client.client.Client): | |||
layout = streams[0][1] | |||
return layout_to_dtype(layout) | |||
def stream_extract_numpy(self, path, start = None, end = None, | |||
layout = None, maxrows = 100000, | |||
structured = False): | |||
def stream_extract_numpy(self, path, start=None, end=None, | |||
layout=None, maxrows=100000, | |||
structured=False): | |||
""" | |||
Extract data from a stream. Returns a generator that yields | |||
Numpy arrays of up to 'maxrows' of data each. | |||
@@ -67,7 +67,7 @@ class NumpyClient(nilmdb.client.client.Client): | |||
chunks = [] | |||
total_len = 0 | |||
maxsize = dtype.itemsize * maxrows | |||
for data in self.stream_extract(path, start, end, binary = True): | |||
for data in self.stream_extract(path, start, end, binary=True): | |||
# Add this block of binary data | |||
chunks.append(data) | |||
total_len += len(data) | |||
@@ -76,7 +76,7 @@ class NumpyClient(nilmdb.client.client.Client): | |||
while total_len >= maxsize: | |||
assembled = b"".join(chunks) | |||
total_len -= maxsize | |||
chunks = [ assembled[maxsize:] ] | |||
chunks = [assembled[maxsize:]] | |||
block = assembled[:maxsize] | |||
yield to_numpy(block) | |||
@@ -84,8 +84,8 @@ class NumpyClient(nilmdb.client.client.Client): | |||
yield to_numpy(b"".join(chunks)) | |||
@contextlib.contextmanager | |||
def stream_insert_numpy_context(self, path, start = None, end = None, | |||
layout = None): | |||
def stream_insert_numpy_context(self, path, start=None, end=None, | |||
layout=None): | |||
"""Return a context manager that allows data to be efficiently | |||
inserted into a stream in a piecewise manner. Data is | |||
provided as Numpy arrays, and is aggregated and sent to the | |||
@@ -104,8 +104,8 @@ class NumpyClient(nilmdb.client.client.Client): | |||
ctx.finalize() | |||
ctx.destroy() | |||
def stream_insert_numpy(self, path, data, start = None, end = None, | |||
layout = None): | |||
def stream_insert_numpy(self, path, data, start=None, end=None, | |||
layout=None): | |||
"""Insert data into a stream. data should be a Numpy array | |||
which will be passed through stream_insert_numpy_context to | |||
break it into chunks etc. See the help for that function | |||
@@ -118,6 +118,7 @@ class NumpyClient(nilmdb.client.client.Client): | |||
ctx.insert(chunk) | |||
return ctx.last_response | |||
class StreamInserterNumpy(nilmdb.client.client.StreamInserter): | |||
"""Object returned by stream_insert_numpy_context() that manages | |||
the insertion of rows of data into a particular path. | |||
@@ -160,9 +161,9 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter): | |||
# Convert to structured array | |||
sarray = numpy.zeros(array.shape[0], dtype=self._dtype) | |||
try: | |||
sarray['timestamp'] = array[:,0] | |||
sarray['timestamp'] = array[:, 0] | |||
# Need the squeeze in case sarray['data'] is 1 dimensional | |||
sarray['data'] = numpy.squeeze(array[:,1:]) | |||
sarray['data'] = numpy.squeeze(array[:, 1:]) | |||
except (IndexError, ValueError): | |||
raise ValueError("wrong number of fields for this data type") | |||
array = sarray | |||
@@ -188,15 +189,15 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter): | |||
# Send if it's too long | |||
if self._block_rows >= maxrows: | |||
self._send_block(final = False) | |||
self._send_block(final=False) | |||
def _send_block(self, final = False): | |||
def _send_block(self, final=False): | |||
"""Send the data current stored up. One row might be left | |||
over if we need its timestamp saved.""" | |||
# Build the full array to send | |||
if self._block_rows == 0: | |||
array = numpy.zeros(0, dtype = self._dtype) | |||
array = numpy.zeros(0, dtype=self._dtype) | |||
else: | |||
array = numpy.hstack(self._block_arrays) | |||
@@ -207,7 +208,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter): | |||
try: | |||
start_ts = array['timestamp'][0] | |||
except IndexError: | |||
pass # no timestamp is OK, if we have no data | |||
pass # no timestamp is OK, if we have no data | |||
# Get ending timestamp | |||
if final: | |||
@@ -220,7 +221,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter): | |||
end_ts = array['timestamp'][-1] | |||
end_ts += nilmdb.utils.time.epsilon | |||
except IndexError: | |||
pass # no timestamp is OK, if we have no data | |||
pass # no timestamp is OK, if we have no data | |||
self._block_arrays = [] | |||
self._block_rows = 0 | |||
@@ -240,7 +241,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter): | |||
# the server complain so that the error is the same | |||
# as if we hadn't done this chunking. | |||
end_ts = self._interval_end | |||
self._block_arrays = [ array[-1:] ] | |||
self._block_arrays = [array[-1:]] | |||
self._block_rows = 1 | |||
array = array[:-1] | |||
@@ -257,6 +258,6 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter): | |||
# Send it | |||
data = array.tostring() | |||
self.last_response = self._client.stream_insert_block( | |||
self._path, data, start_ts, end_ts, binary = True) | |||
self._path, data, start_ts, end_ts, binary=True) | |||
return |
@@ -2,7 +2,7 @@ | |||
import nilmdb.client | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import fprintf, sprintf | |||
import datetime_tz | |||
import nilmdb.utils.time | |||
@@ -16,13 +16,14 @@ import argcomplete | |||
# Valid subcommands. Defined in separate files just to break | |||
# things up -- they're still called with Cmdline as self. | |||
subcommands = [ "help", "info", "create", "rename", "list", "intervals", | |||
"metadata", "insert", "extract", "remove", "destroy" ] | |||
subcommands = ["help", "info", "create", "rename", "list", "intervals", | |||
"metadata", "insert", "extract", "remove", "destroy"] | |||
# Import the subcommand modules | |||
subcmd_mods = {} | |||
for cmd in subcommands: | |||
subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist = [ cmd ]) | |||
subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist=[cmd]) | |||
class JimArgumentParser(argparse.ArgumentParser): | |||
def parse_args(self, args=None, namespace=None): | |||
@@ -30,18 +31,19 @@ class JimArgumentParser(argparse.ArgumentParser): | |||
# --version". This makes "nilmtool cmd --version" work, which | |||
# is needed by help2man. | |||
if "--version" in (args or sys.argv[1:]): | |||
args = [ "--version" ] | |||
args = ["--version"] | |||
return argparse.ArgumentParser.parse_args(self, args, namespace) | |||
def error(self, message): | |||
self.print_usage(sys.stderr) | |||
self.exit(2, sprintf("error: %s\n", message)) | |||
class Complete(object): | |||
# Completion helpers, for using argcomplete (see | |||
# extras/nilmtool-bash-completion.sh) | |||
def escape(self, s): | |||
quote_chars = [ "\\", "\"", "'", " " ] | |||
quote_chars = ["\\", "\"", "'", " "] | |||
for char in quote_chars: | |||
s = s.replace(char, "\\" + char) | |||
return s | |||
@@ -54,18 +56,18 @@ class Complete(object): | |||
def path(self, prefix, parsed_args, **kwargs): | |||
client = nilmdb.client.Client(parsed_args.url) | |||
return ( self.escape(s[0]) | |||
for s in client.stream_list() | |||
if s[0].startswith(prefix) ) | |||
return (self.escape(s[0]) | |||
for s in client.stream_list() | |||
if s[0].startswith(prefix)) | |||
def layout(self, prefix, parsed_args, **kwargs): | |||
types = [ "int8", "int16", "int32", "int64", | |||
"uint8", "uint16", "uint32", "uint64", | |||
"float32", "float64" ] | |||
types = ["int8", "int16", "int32", "int64", | |||
"uint8", "uint16", "uint32", "uint64", | |||
"float32", "float64"] | |||
layouts = [] | |||
for i in range(1,10): | |||
for i in range(1, 10): | |||
layouts.extend([(t + "_" + str(i)) for t in types]) | |||
return ( l for l in layouts if l.startswith(prefix) ) | |||
return (l for l in layouts if l.startswith(prefix)) | |||
def meta_key(self, prefix, parsed_args, **kwargs): | |||
return (kv.split('=')[0] for kv | |||
@@ -77,21 +79,22 @@ class Complete(object): | |||
if not path: | |||
return [] | |||
results = [] | |||
for (k,v) in client.stream_get_metadata(path).items(): | |||
for (k, v) in client.stream_get_metadata(path).items(): | |||
kv = self.escape(k + '=' + v) | |||
if kv.startswith(prefix): | |||
results.append(kv) | |||
return results | |||
class Cmdline(object): | |||
def __init__(self, argv = None): | |||
def __init__(self, argv=None): | |||
self.argv = argv or sys.argv[1:] | |||
self.client = None | |||
self.def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/") | |||
self.subcmd = {} | |||
self.complete = Complete() | |||
self.complete_output_stream = None # overridden by test suite | |||
self.complete_output_stream = None # overridden by test suite | |||
def arg_time(self, toparse): | |||
"""Parse a time string argument""" | |||
@@ -103,14 +106,14 @@ class Cmdline(object): | |||
# Set up the parser | |||
def parser_setup(self): | |||
self.parser = JimArgumentParser(add_help = False, | |||
formatter_class = def_form) | |||
self.parser = JimArgumentParser(add_help=False, | |||
formatter_class=def_form) | |||
group = self.parser.add_argument_group("General options") | |||
group.add_argument("-h", "--help", action='help', | |||
help='show this help message and exit') | |||
group.add_argument("-v", "--version", action="version", | |||
version = nilmdb.__version__) | |||
version=nilmdb.__version__) | |||
group = self.parser.add_argument_group("Server") | |||
group.add_argument("-u", "--url", action="store", | |||
@@ -158,7 +161,7 @@ class Cmdline(object): | |||
# unless the particular command requests that we don't. | |||
if "no_test_connect" not in self.args: | |||
try: | |||
server_version = self.client.version() | |||
self.client.version() | |||
except nilmdb.client.Error as e: | |||
self.die("error connecting to server: %s", str(e)) | |||
@@ -1,11 +1,11 @@ | |||
from nilmdb.utils.printf import * | |||
import nilmdb.client | |||
from argparse import RawDescriptionHelpFormatter as raw_form | |||
def setup(self, sub): | |||
cmd = sub.add_parser("create", help="Create a new stream", | |||
formatter_class = raw_form, | |||
formatter_class=raw_form, | |||
description=""" | |||
Create a new empty stream at the specified path and with the specified | |||
layout type. | |||
@@ -19,7 +19,7 @@ Layout types are of the format: type_count | |||
For example, 'float32_8' means the data for this stream has 8 columns of | |||
32-bit floating point values. | |||
""") | |||
cmd.set_defaults(handler = cmd_create) | |||
cmd.set_defaults(handler=cmd_create) | |||
group = cmd.add_argument_group("Required arguments") | |||
group.add_argument("path", | |||
help="Path (in database) of new stream, e.g. /foo/bar", | |||
@@ -29,6 +29,7 @@ Layout types are of the format: type_count | |||
).completer = self.complete.layout | |||
return cmd | |||
def cmd_create(self): | |||
"""Create new stream""" | |||
try: | |||
@@ -1,12 +1,13 @@ | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
import nilmdb.client | |||
import fnmatch | |||
from argparse import ArgumentDefaultsHelpFormatter as def_form | |||
def setup(self, sub): | |||
cmd = sub.add_parser("destroy", help="Delete a stream and all data", | |||
formatter_class = def_form, | |||
formatter_class=def_form, | |||
description=""" | |||
Destroy the stream at the specified path. | |||
The stream must be empty. All metadata | |||
@@ -14,7 +15,7 @@ def setup(self, sub): | |||
Wildcards and multiple paths are supported. | |||
""") | |||
cmd.set_defaults(handler = cmd_destroy) | |||
cmd.set_defaults(handler=cmd_destroy) | |||
group = cmd.add_argument_group("Options") | |||
group.add_argument("-R", "--remove", action="store_true", | |||
help="Remove all data before destroying stream") | |||
@@ -27,9 +28,10 @@ def setup(self, sub): | |||
).completer = self.complete.path | |||
return cmd | |||
def cmd_destroy(self): | |||
"""Destroy stream""" | |||
streams = [ s[0] for s in self.client.stream_list() ] | |||
streams = [s[0] for s in self.client.stream_list()] | |||
paths = [] | |||
for path in self.args.path: | |||
new = fnmatch.filter(streams, path) | |||
@@ -43,7 +45,7 @@ def cmd_destroy(self): | |||
try: | |||
if self.args.remove: | |||
count = self.client.stream_remove(path) | |||
self.client.stream_remove(path) | |||
self.client.stream_destroy(path) | |||
except nilmdb.client.ClientError as e: | |||
self.die("error destroying stream: %s", str(e)) |
@@ -1,15 +1,15 @@ | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
import nilmdb.client | |||
import sys | |||
def setup(self, sub): | |||
cmd = sub.add_parser("extract", help="Extract data", | |||
description=""" | |||
Extract data from a stream. | |||
""") | |||
cmd.set_defaults(verify = cmd_extract_verify, | |||
handler = cmd_extract) | |||
cmd.set_defaults(verify=cmd_extract_verify, | |||
handler=cmd_extract) | |||
group = cmd.add_argument_group("Data selection") | |||
group.add_argument("path", | |||
@@ -40,15 +40,17 @@ def setup(self, sub): | |||
help="Just output a count of matched data points") | |||
return cmd | |||
def cmd_extract_verify(self): | |||
if self.args.start > self.args.end: | |||
self.parser.error("start is after end") | |||
if self.args.binary: | |||
if (self.args.bare or self.args.annotate or self.args.markup or | |||
self.args.timestamp_raw or self.args.count): | |||
self.args.timestamp_raw or self.args.count): | |||
self.parser.error("--binary cannot be combined with other options") | |||
def cmd_extract(self): | |||
streams = self.client.stream_list(self.args.path) | |||
if len(streams) != 1: | |||
@@ -1,7 +1,5 @@ | |||
from nilmdb.utils.printf import * | |||
import argparse | |||
import sys | |||
def setup(self, sub): | |||
cmd = sub.add_parser("help", help="Show detailed help for a command", | |||
@@ -9,14 +7,15 @@ def setup(self, sub): | |||
Show help for a command. 'help command' is | |||
the same as 'command --help'. | |||
""") | |||
cmd.set_defaults(handler = cmd_help) | |||
cmd.set_defaults(no_test_connect = True) | |||
cmd.set_defaults(handler=cmd_help) | |||
cmd.set_defaults(no_test_connect=True) | |||
cmd.add_argument("command", nargs="?", | |||
help="Command to get help about") | |||
cmd.add_argument("rest", nargs=argparse.REMAINDER, | |||
help=argparse.SUPPRESS) | |||
return cmd | |||
def cmd_help(self): | |||
if self.args.command in self.subcmd: | |||
self.subcmd[self.args.command].print_help() | |||
@@ -1,19 +1,21 @@ | |||
import nilmdb.client | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
from nilmdb.utils import human_size | |||
from argparse import ArgumentDefaultsHelpFormatter as def_form | |||
def setup(self, sub): | |||
cmd = sub.add_parser("info", help="Server information", | |||
formatter_class = def_form, | |||
formatter_class=def_form, | |||
description=""" | |||
List information about the server, like | |||
version. | |||
""") | |||
cmd.set_defaults(handler = cmd_info) | |||
cmd.set_defaults(handler=cmd_info) | |||
return cmd | |||
def cmd_info(self): | |||
"""Print info about the server""" | |||
printf("Client version: %s\n", nilmdb.__version__) | |||
@@ -1,17 +1,18 @@ | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
import nilmdb.client | |||
import nilmdb.utils.timestamper as timestamper | |||
import nilmdb.utils.time | |||
import sys | |||
def setup(self, sub): | |||
cmd = sub.add_parser("insert", help="Insert data", | |||
description=""" | |||
Insert data into a stream. | |||
""") | |||
cmd.set_defaults(verify = cmd_insert_verify, | |||
handler = cmd_insert) | |||
cmd.set_defaults(verify=cmd_insert_verify, | |||
handler=cmd_insert) | |||
cmd.add_argument("-q", "--quiet", action='store_true', | |||
help='suppress unnecessary messages') | |||
@@ -61,21 +62,24 @@ def setup(self, sub): | |||
group.add_argument("path", | |||
help="Path of stream, e.g. /foo/bar", | |||
).completer = self.complete.path | |||
group.add_argument("file", nargs = '?', default='-', | |||
group.add_argument("file", nargs='?', default='-', | |||
help="File to insert (default: - (stdin))") | |||
return cmd | |||
def cmd_insert_verify(self): | |||
if self.args.timestamp: | |||
if not self.args.rate: | |||
self.die("error: --rate is needed, but was not specified") | |||
if not self.args.filename and self.args.start is None: | |||
self.die("error: need --start or --filename when adding timestamps") | |||
self.die("error: need --start or --filename " | |||
"when adding timestamps") | |||
else: | |||
if self.args.start is None or self.args.end is None: | |||
self.die("error: when not adding timestamps, --start and " | |||
"--end are required") | |||
def cmd_insert(self): | |||
# Find requested stream | |||
streams = self.client.stream_list(self.args.path) | |||
@@ -1,14 +1,13 @@ | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
import nilmdb.utils.time | |||
from nilmdb.utils.interval import Interval | |||
import fnmatch | |||
import argparse | |||
from argparse import ArgumentDefaultsHelpFormatter as def_form | |||
def setup(self, sub): | |||
cmd = sub.add_parser("intervals", help="List intervals", | |||
formatter_class = def_form, | |||
formatter_class=def_form, | |||
description=""" | |||
List intervals in a stream, similar to | |||
'list --detail path'. | |||
@@ -17,8 +16,8 @@ def setup(self, sub): | |||
interval ranges that are present in 'path' | |||
and not present in 'diffpath' are printed. | |||
""") | |||
cmd.set_defaults(verify = cmd_intervals_verify, | |||
handler = cmd_intervals) | |||
cmd.set_defaults(verify=cmd_intervals_verify, | |||
handler=cmd_intervals) | |||
group = cmd.add_argument_group("Stream selection") | |||
group.add_argument("path", metavar="PATH", | |||
@@ -48,11 +47,13 @@ def setup(self, sub): | |||
return cmd | |||
def cmd_intervals_verify(self): | |||
if self.args.start is not None and self.args.end is not None: | |||
if self.args.start >= self.args.end: | |||
self.parser.error("start must precede end") | |||
def cmd_intervals(self): | |||
"""List intervals in a stream""" | |||
if self.args.timestamp_raw: | |||
@@ -61,11 +62,11 @@ def cmd_intervals(self): | |||
time_string = nilmdb.utils.time.timestamp_to_human | |||
try: | |||
intervals = ( Interval(start, end) for (start, end) in | |||
self.client.stream_intervals(self.args.path, | |||
self.args.start, | |||
self.args.end, | |||
self.args.diff) ) | |||
intervals = (Interval(start, end) for (start, end) in | |||
self.client.stream_intervals(self.args.path, | |||
self.args.start, | |||
self.args.end, | |||
self.args.diff)) | |||
if self.args.optimize: | |||
intervals = nilmdb.utils.interval.optimize(intervals) | |||
for i in intervals: | |||
@@ -73,4 +74,3 @@ def cmd_intervals(self): | |||
except nilmdb.client.ClientError as e: | |||
self.die("error listing intervals: %s", str(e)) | |||
@@ -1,21 +1,21 @@ | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
import nilmdb.utils.time | |||
import fnmatch | |||
import argparse | |||
from argparse import ArgumentDefaultsHelpFormatter as def_form | |||
def setup(self, sub): | |||
cmd = sub.add_parser("list", help="List streams", | |||
formatter_class = def_form, | |||
formatter_class=def_form, | |||
description=""" | |||
List streams available in the database, | |||
optionally filtering by path. Wildcards | |||
are accepted; non-matching paths or wildcards | |||
are ignored. | |||
""") | |||
cmd.set_defaults(verify = cmd_list_verify, | |||
handler = cmd_list) | |||
cmd.set_defaults(verify=cmd_list_verify, | |||
handler=cmd_list) | |||
group = cmd.add_argument_group("Stream filtering") | |||
group.add_argument("path", metavar="PATH", default=["*"], nargs='*', | |||
@@ -50,6 +50,7 @@ def setup(self, sub): | |||
return cmd | |||
def cmd_list_verify(self): | |||
if self.args.start is not None and self.args.end is not None: | |||
if self.args.start >= self.args.end: | |||
@@ -57,11 +58,13 @@ def cmd_list_verify(self): | |||
if self.args.start is not None or self.args.end is not None: | |||
if not self.args.detail: | |||
self.parser.error("--start and --end only make sense with --detail") | |||
self.parser.error("--start and --end only make sense " | |||
"with --detail") | |||
def cmd_list(self): | |||
"""List available streams""" | |||
streams = self.client.stream_list(extended = True) | |||
streams = self.client.stream_list(extended=True) | |||
if self.args.timestamp_raw: | |||
time_string = nilmdb.utils.time.timestamp_to_string | |||
@@ -94,7 +97,7 @@ def cmd_list(self): | |||
if self.args.detail: | |||
printed = False | |||
for (start, end) in self.client.stream_intervals( | |||
path, self.args.start, self.args.end): | |||
path, self.args.start, self.args.end): | |||
printf(" [ %s -> %s ]\n", | |||
time_string(start), time_string(end)) | |||
printed = True | |||
@@ -1,7 +1,8 @@ | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
import nilmdb | |||
import nilmdb.client | |||
def setup(self, sub): | |||
cmd = sub.add_parser("metadata", help="Get or set stream metadata", | |||
description=""" | |||
@@ -11,7 +12,7 @@ def setup(self, sub): | |||
usage="%(prog)s path [-g [key ...] | " | |||
"-s key=value [...] | -u key=value [...]] | " | |||
"-d [key ...]") | |||
cmd.set_defaults(handler = cmd_metadata) | |||
cmd.set_defaults(handler=cmd_metadata) | |||
group = cmd.add_argument_group("Required arguments") | |||
group.add_argument("path", | |||
@@ -36,6 +37,7 @@ def setup(self, sub): | |||
).completer = self.complete.meta_key | |||
return cmd | |||
def cmd_metadata(self): | |||
"""Manipulate metadata""" | |||
if self.args.set is not None or self.args.update is not None: | |||
@@ -1,15 +1,17 @@ | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
import nilmdb.client | |||
import fnmatch | |||
def setup(self, sub): | |||
cmd = sub.add_parser("remove", help="Remove data", | |||
description=""" | |||
Remove all data from a specified time range within a | |||
stream. If multiple streams or wildcards are provided, | |||
the same time range is removed from all streams. | |||
stream. If multiple streams or wildcards are | |||
provided, the same time range is removed from all | |||
streams. | |||
""") | |||
cmd.set_defaults(handler = cmd_remove) | |||
cmd.set_defaults(handler=cmd_remove) | |||
group = cmd.add_argument_group("Data selection") | |||
group.add_argument("path", nargs='+', | |||
@@ -32,8 +34,9 @@ def setup(self, sub): | |||
help="Output number of data points removed") | |||
return cmd | |||
def cmd_remove(self): | |||
streams = [ s[0] for s in self.client.stream_list() ] | |||
streams = [s[0] for s in self.client.stream_list()] | |||
paths = [] | |||
for path in self.args.path: | |||
new = fnmatch.filter(streams, path) | |||
@@ -48,7 +51,7 @@ def cmd_remove(self): | |||
count = self.client.stream_remove(path, | |||
self.args.start, self.args.end) | |||
if self.args.count: | |||
printf("%d\n", count); | |||
printf("%d\n", count) | |||
except nilmdb.client.ClientError as e: | |||
self.die("error removing data: %s", str(e)) | |||
@@ -1,18 +1,18 @@ | |||
from nilmdb.utils.printf import * | |||
import nilmdb.client | |||
from argparse import ArgumentDefaultsHelpFormatter as def_form | |||
def setup(self, sub): | |||
cmd = sub.add_parser("rename", help="Rename a stream", | |||
formatter_class = def_form, | |||
formatter_class=def_form, | |||
description=""" | |||
Rename a stream. | |||
Only the stream's path is renamed; no | |||
metadata is changed. | |||
""") | |||
cmd.set_defaults(handler = cmd_rename) | |||
cmd.set_defaults(handler=cmd_rename) | |||
group = cmd.add_argument_group("Required arguments") | |||
group.add_argument("oldpath", | |||
help="Old path, e.g. /foo/old", | |||
@@ -23,6 +23,7 @@ def setup(self, sub): | |||
return cmd | |||
def cmd_rename(self): | |||
"""Rename a stream""" | |||
try: | |||
@@ -1,5 +1,3 @@ | |||
"""nilmdb.fsck""" | |||
from nilmdb.fsck.fsck import Fsck |
@@ -12,7 +12,7 @@ import nilmdb.server | |||
import nilmdb.client.numpyclient | |||
from nilmdb.utils.interval import IntervalError | |||
from nilmdb.server.interval import Interval, IntervalSet | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf, fprintf, sprintf | |||
from nilmdb.utils.time import timestamp_to_string | |||
from collections import defaultdict | |||
@@ -7,34 +7,35 @@ import socket | |||
import cherrypy | |||
import sys | |||
def main(): | |||
"""Main entry point for the 'nilmdb-server' command line script""" | |||
parser = argparse.ArgumentParser( | |||
description = 'Run the NilmDB server', | |||
formatter_class = argparse.ArgumentDefaultsHelpFormatter) | |||
description='Run the NilmDB server', | |||
formatter_class=argparse.ArgumentDefaultsHelpFormatter) | |||
parser.add_argument("-v", "--version", action="version", | |||
version = nilmdb.__version__) | |||
version=nilmdb.__version__) | |||
group = parser.add_argument_group("Standard options") | |||
group.add_argument('-a', '--address', | |||
help = 'Only listen on the given address', | |||
default = '0.0.0.0') | |||
group.add_argument('-p', '--port', help = 'Listen on the given port', | |||
type = int, default = 12380) | |||
group.add_argument('-d', '--database', help = 'Database directory', | |||
default = "./db") | |||
group.add_argument('-q', '--quiet', help = 'Silence output', | |||
action = 'store_true') | |||
help='Only listen on the given address', | |||
default='0.0.0.0') | |||
group.add_argument('-p', '--port', help='Listen on the given port', | |||
type=int, default=12380) | |||
group.add_argument('-d', '--database', help='Database directory', | |||
default="./db") | |||
group.add_argument('-q', '--quiet', help='Silence output', | |||
action='store_true') | |||
group.add_argument('-t', '--traceback', | |||
help = 'Provide tracebacks in client errors', | |||
action = 'store_true', default = False) | |||
help='Provide tracebacks in client errors', | |||
action='store_true', default=False) | |||
group = parser.add_argument_group("Debug options") | |||
group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and ' | |||
group.add_argument('-y', '--yappi', help='Run under yappi profiler and ' | |||
'invoke interactive shell afterwards', | |||
action = 'store_true') | |||
action='store_true') | |||
args = parser.parse_args() | |||
@@ -45,10 +46,11 @@ def main(): | |||
# Configure the server | |||
if not args.quiet: | |||
cherrypy._cpconfig.environments['embedded']['log.screen'] = True | |||
server = nilmdb.server.Server(db, | |||
host = args.address, | |||
port = args.port, | |||
force_traceback = args.traceback) | |||
host=args.address, | |||
port=args.port, | |||
force_traceback=args.traceback) | |||
# Print info | |||
if not args.quiet: | |||
@@ -58,7 +60,7 @@ def main(): | |||
host = socket.getfqdn() | |||
else: | |||
host = args.address | |||
print("Server URL: http://%s:%d/" % ( host, args.port)) | |||
print("Server URL: http://%s:%d/" % (host, args.port)) | |||
print("----") | |||
# Run it | |||
@@ -68,18 +70,18 @@ def main(): | |||
try: | |||
import yappi | |||
yappi.start() | |||
server.start(blocking = True) | |||
server.start(blocking=True) | |||
finally: | |||
yappi.stop() | |||
stats = yappi.get_func_stats() | |||
stats.sort("ttot") | |||
stats.print_all() | |||
from IPython import embed | |||
embed(header = "Use the `yappi` or `stats` object to explore " | |||
embed(header="Use the `yappi` or `stats` object to explore " | |||
"further, quit to exit") | |||
else: | |||
server.start(blocking = True) | |||
except nilmdb.server.serverutil.CherryPyExit as e: | |||
server.start(blocking=True) | |||
except nilmdb.server.serverutil.CherryPyExit: | |||
print("Exiting due to CherryPy error", file=sys.stderr) | |||
raise | |||
finally: | |||
@@ -87,5 +89,6 @@ def main(): | |||
print("Closing database") | |||
db.close() | |||
if __name__ == "__main__": | |||
main() |
@@ -2,9 +2,11 @@ | |||
import nilmdb.cmdline | |||
def main(): | |||
"""Main entry point for the 'nilmtool' command line script""" | |||
nilmdb.cmdline.Cmdline().run() | |||
if __name__ == "__main__": | |||
main() |
@@ -2,7 +2,7 @@ | |||
# Set up pyximport to automatically rebuild Cython modules if needed. | |||
import pyximport | |||
pyximport.install(inplace = True, build_in_temp = False) | |||
pyximport.install(inplace=True, build_in_temp=False) | |||
from nilmdb.server.nilmdb import NilmDB | |||
from nilmdb.server.server import Server, wsgi_application | |||
@@ -4,8 +4,8 @@ | |||
# nilmdb.py, but will pull the parent nilmdb module instead. | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.time import timestamp_to_string as timestamp_to_string | |||
from nilmdb.utils.printf import sprintf | |||
from nilmdb.utils.time import timestamp_to_string | |||
import nilmdb.utils | |||
import os | |||
@@ -22,7 +22,8 @@ from . import rocket | |||
table_cache_size = 32 | |||
fd_cache_size = 8 | |||
@nilmdb.utils.must_close(wrap_verify = False) | |||
@nilmdb.utils.must_close(wrap_verify=False) | |||
class BulkData(object): | |||
def __init__(self, basepath, **kwargs): | |||
if isinstance(basepath, str): | |||
@@ -103,7 +104,7 @@ class BulkData(object): | |||
if path[0:1] != b'/': | |||
raise ValueError("paths must start with / ") | |||
[ group, node ] = path.rsplit(b"/", 1) | |||
[group, node] = path.rsplit(b"/", 1) | |||
if group == b'': | |||
raise ValueError("invalid path; path must contain at least one " | |||
"folder") | |||
@@ -129,7 +130,7 @@ class BulkData(object): | |||
if not os.path.isdir(ospath): | |||
os.mkdir(ospath) | |||
made_dirs.append(ospath) | |||
except Exception as e: | |||
except Exception: | |||
# Remove paths that we created | |||
for ospath in reversed(made_dirs): | |||
os.rmdir(ospath) | |||
@@ -204,7 +205,7 @@ class BulkData(object): | |||
self.getnode.cache_remove(self, oldunicodepath) | |||
# Move the table to a temporary location | |||
tmpdir = tempfile.mkdtemp(prefix = b"rename-", dir = self.root) | |||
tmpdir = tempfile.mkdtemp(prefix=b"rename-", dir=self.root) | |||
tmppath = os.path.join(tmpdir, b"table") | |||
os.rename(oldospath, tmppath) | |||
@@ -242,7 +243,7 @@ class BulkData(object): | |||
# Remove the contents of the target directory | |||
if not Table.exists(ospath): | |||
raise ValueError("nothing at that path") | |||
for (root, dirs, files) in os.walk(ospath, topdown = False): | |||
for (root, dirs, files) in os.walk(ospath, topdown=False): | |||
for name in files: | |||
os.remove(os.path.join(root, name)) | |||
for name in dirs: | |||
@@ -252,8 +253,8 @@ class BulkData(object): | |||
self._remove_leaves(unicodepath) | |||
# Cache open tables | |||
@nilmdb.utils.lru_cache(size = table_cache_size, | |||
onremove = lambda x: x.close()) | |||
@nilmdb.utils.lru_cache(size=table_cache_size, | |||
onremove=lambda x: x.close()) | |||
def getnode(self, unicodepath): | |||
"""Return a Table object corresponding to the given database | |||
path, which must exist.""" | |||
@@ -262,7 +263,8 @@ class BulkData(object): | |||
ospath = os.path.join(self.root, *elements) | |||
return Table(ospath, self.initial_nrows) | |||
@nilmdb.utils.must_close(wrap_verify = False) | |||
@nilmdb.utils.must_close(wrap_verify=False) | |||
class Table(object): | |||
"""Tools to help access a single table (data at a specific OS path).""" | |||
# See design.md for design details | |||
@@ -289,15 +291,17 @@ class Table(object): | |||
rows_per_file = max(file_size // rkt.binary_size, 1) | |||
rkt.close() | |||
fmt = { "rows_per_file": rows_per_file, | |||
"files_per_dir": files_per_dir, | |||
"layout": layout, | |||
"version": 3 } | |||
fmt = { | |||
"rows_per_file": rows_per_file, | |||
"files_per_dir": files_per_dir, | |||
"layout": layout, | |||
"version": 3 | |||
} | |||
with open(os.path.join(root, b"_format"), "wb") as f: | |||
pickle.dump(fmt, f, 2) | |||
# Normal methods | |||
def __init__(self, root, initial_nrows = 0): | |||
def __init__(self, root, initial_nrows=0): | |||
"""'root' is the full OS path to the directory of this table""" | |||
self.root = root | |||
self.initial_nrows = initial_nrows | |||
@@ -343,7 +347,7 @@ class Table(object): | |||
# empty if something was deleted but the directory was unexpectedly | |||
# not deleted. | |||
subdirs = sorted(filter(regex.search, os.listdir(self.root)), | |||
key = lambda x: int(x, 16), reverse = True) | |||
key=lambda x: int(x, 16), reverse=True) | |||
for subdir in subdirs: | |||
# Now find the last file in that dir | |||
@@ -354,7 +358,7 @@ class Table(object): | |||
continue | |||
# Find the numerical max | |||
filename = max(files, key = lambda x: int(x, 16)) | |||
filename = max(files, key=lambda x: int(x, 16)) | |||
offset = os.path.getsize(os.path.join(self.root, subdir, filename)) | |||
# Convert to row number | |||
@@ -396,7 +400,7 @@ class Table(object): | |||
row = (filenum * self.rows_per_file) + (offset // self.row_size) | |||
return row | |||
def _remove_or_truncate_file(self, subdir, filename, offset = 0): | |||
def _remove_or_truncate_file(self, subdir, filename, offset=0): | |||
"""Remove the given file, and remove the subdirectory too | |||
if it's empty. If offset is nonzero, truncate the file | |||
to that size instead.""" | |||
@@ -416,8 +420,8 @@ class Table(object): | |||
pass | |||
# Cache open files | |||
@nilmdb.utils.lru_cache(size = fd_cache_size, | |||
onremove = lambda f: f.close()) | |||
@nilmdb.utils.lru_cache(size=fd_cache_size, | |||
onremove=lambda f: f.close()) | |||
def file_open(self, subdir, filename): | |||
"""Open and map a given 'subdir/filename' (relative to self.root). | |||
Will be automatically closed when evicted from the cache.""" | |||
@@ -430,7 +434,7 @@ class Table(object): | |||
return rocket.Rocket(self.layout, | |||
os.path.join(self.root, subdir, filename)) | |||
def append_data(self, data, start, end, binary = False): | |||
def append_data(self, data, start, end, binary=False): | |||
"""Parse the formatted string in 'data', according to the | |||
current layout, and append it to the table. If any timestamps | |||
are non-monotonic, or don't fall between 'start' and 'end', | |||
@@ -454,7 +458,7 @@ class Table(object): | |||
while data_offset < len(data): | |||
# See how many rows we can fit into the current file, | |||
# and open it | |||
(subdir, fname, offset, count) = self._offset_from_row(tot_rows) | |||
(subdir, fname, offs, count) = self._offset_from_row(tot_rows) | |||
f = self.file_open(subdir, fname) | |||
# Ask the rocket object to parse and append up to "count" | |||
@@ -503,8 +507,8 @@ class Table(object): | |||
# deleting files that we may have appended data to. | |||
cleanpos = self.nrows | |||
while cleanpos <= tot_rows: | |||
(subdir, fname, offset, count) = self._offset_from_row(cleanpos) | |||
self._remove_or_truncate_file(subdir, fname, offset) | |||
(subdir, fname, offs, count) = self._offset_from_row(cleanpos) | |||
self._remove_or_truncate_file(subdir, fname, offs) | |||
cleanpos += count | |||
# Re-raise original exception | |||
raise | |||
@@ -512,14 +516,11 @@ class Table(object): | |||
# Success, so update self.nrows accordingly | |||
self.nrows = tot_rows | |||
def get_data(self, start, stop, binary = False): | |||
def get_data(self, start, stop, binary=False): | |||
"""Extract data corresponding to Python range [n:m], | |||
and returns a formatted string""" | |||
if (start is None or | |||
stop is None or | |||
start > stop or | |||
start < 0 or | |||
stop > self.nrows): | |||
if (start is None or stop is None or | |||
start > stop or start < 0 or stop > self.nrows): | |||
raise IndexError("Index out of range") | |||
ret = [] | |||
@@ -597,7 +598,7 @@ class Table(object): | |||
# remainder will be filled on a subsequent append(), and things | |||
# are generally easier if we don't have to special-case that. | |||
if (len(merged) == 1 and | |||
merged[0][0] == 0 and merged[0][1] == self.rows_per_file): | |||
merged[0][0] == 0 and merged[0][1] == self.rows_per_file): | |||
# Delete files | |||
if cachefile_present: | |||
os.remove(cachefile) | |||
@@ -1,12 +1,15 @@ | |||
"""Exceptions""" | |||
class NilmDBError(Exception): | |||
"""Base exception for NilmDB errors""" | |||
def __init__(self, msg = "Unspecified error"): | |||
def __init__(self, msg="Unspecified error"): | |||
super().__init__(msg) | |||
class StreamError(NilmDBError): | |||
pass | |||
class OverlapError(NilmDBError): | |||
pass |
@@ -11,7 +11,7 @@ Manages both the SQL database and the table storage backend. | |||
# nilmdb.py, but will pull the parent nilmdb module instead. | |||
import nilmdb.utils | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import printf | |||
from nilmdb.utils.time import timestamp_to_bytes | |||
from nilmdb.utils.interval import IntervalError | |||
@@ -37,7 +37,7 @@ import errno | |||
# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL' | |||
# give an equivalent speedup more safely. That is what is used here. | |||
_sql_schema_updates = { | |||
0: { "next": 1, "sql": """ | |||
0: {"next": 1, "sql": """ | |||
-- All streams | |||
CREATE TABLE streams( | |||
id INTEGER PRIMARY KEY, -- stream ID | |||
@@ -61,23 +61,24 @@ _sql_schema_updates = { | |||
end_pos INTEGER NOT NULL | |||
); | |||
CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time); | |||
""" }, | |||
"""}, | |||
1: { "next": 3, "sql": """ | |||
1: {"next": 3, "sql": """ | |||
-- Generic dictionary-type metadata that can be associated with a stream | |||
CREATE TABLE metadata( | |||
stream_id INTEGER NOT NULL, | |||
stream_id INTEGER NOT NULL, | |||
key TEXT NOT NULL, | |||
value TEXT | |||
); | |||
""" }, | |||
"""}, | |||
2: { "error": "old format with floating-point timestamps requires " | |||
"nilmdb 1.3.1 or older" }, | |||
2: {"error": "old format with floating-point timestamps requires " | |||
"nilmdb 1.3.1 or older"}, | |||
3: { "next": None }, | |||
3: {"next": None}, | |||
} | |||
@nilmdb.utils.must_close() | |||
class NilmDB(object): | |||
verbose = 0 | |||
@@ -119,7 +120,7 @@ class NilmDB(object): | |||
# SQLite database too | |||
sqlfilename = os.path.join(self.basepath, "data.sql") | |||
self.con = sqlite3.connect(sqlfilename, check_same_thread = True) | |||
self.con = sqlite3.connect(sqlfilename, check_same_thread=True) | |||
try: | |||
self._sql_schema_update() | |||
except Exception: | |||
@@ -183,7 +184,7 @@ class NilmDB(object): | |||
raise NilmDBError("start must precede end") | |||
return (start, end) | |||
@nilmdb.utils.lru_cache(size = 64) | |||
@nilmdb.utils.lru_cache(size=64) | |||
def _get_intervals(self, stream_id): | |||
""" | |||
Return a mutable IntervalSet corresponding to the given stream ID. | |||
@@ -231,11 +232,11 @@ class NilmDB(object): | |||
# time range [adjacent.start -> interval.end) | |||
# and database rows [ adjacent.start_pos -> end_pos ]. | |||
# Only do this if the resulting interval isn't too large. | |||
max_merged_rows = 8000 * 60 * 60 * 1.05 # 1.05 hours at 8 KHz | |||
max_merged_rows = 8000 * 60 * 60 * 1.05 # 1.05 hours at 8 KHz | |||
adjacent = iset.find_end(interval.start) | |||
if (adjacent is not None and | |||
start_pos == adjacent.db_endpos and | |||
(end_pos - adjacent.db_startpos) < max_merged_rows): | |||
start_pos == adjacent.db_endpos and | |||
(end_pos - adjacent.db_startpos) < max_merged_rows): | |||
# First delete the old one, both from our iset and the | |||
# database | |||
iset -= adjacent | |||
@@ -281,7 +282,8 @@ class NilmDB(object): | |||
# the removed piece was in the middle. | |||
def add(iset, start, end, start_pos, end_pos): | |||
iset += DBInterval(start, end, start, end, start_pos, end_pos) | |||
self._sql_interval_insert(stream_id, start, end, start_pos, end_pos) | |||
self._sql_interval_insert(stream_id, start, end, | |||
start_pos, end_pos) | |||
if original.start != remove.start: | |||
# Interval before the removed region | |||
@@ -298,7 +300,7 @@ class NilmDB(object): | |||
return | |||
def stream_list(self, path = None, layout = None, extended = False): | |||
def stream_list(self, path=None, layout=None, extended=False): | |||
"""Return list of lists of all streams in the database. | |||
If path is specified, include only streams with a path that | |||
@@ -307,10 +309,10 @@ class NilmDB(object): | |||
If layout is specified, include only streams with a layout | |||
that matches the given string. | |||
If extended = False, returns a list of lists containing | |||
If extended=False, returns a list of lists containing | |||
the path and layout: [ path, layout ] | |||
If extended = True, returns a list of lists containing | |||
If extended=True, returns a list of lists containing | |||
more information: | |||
path | |||
layout | |||
@@ -337,9 +339,9 @@ class NilmDB(object): | |||
params += (path,) | |||
query += " GROUP BY streams.id ORDER BY streams.path" | |||
result = self.con.execute(query, params).fetchall() | |||
return [ list(x) for x in result ] | |||
return [list(x) for x in result] | |||
def stream_intervals(self, path, start = None, end = None, diffpath = None): | |||
def stream_intervals(self, path, start=None, end=None, diffpath=None): | |||
""" | |||
List all intervals in 'path' between 'start' and 'end'. If | |||
'diffpath' is not none, list instead the set-difference | |||
@@ -411,8 +413,8 @@ class NilmDB(object): | |||
def stream_set_metadata(self, path, data): | |||
"""Set stream metadata from a dictionary, e.g. | |||
{ description = 'Downstairs lighting', | |||
v_scaling = 123.45 } | |||
{ description: 'Downstairs lighting', | |||
v_scaling: 123.45 } | |||
This replaces all existing metadata. | |||
""" | |||
stream_id = self._stream_id(path) | |||
@@ -474,7 +476,7 @@ class NilmDB(object): | |||
con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,)) | |||
con.execute("DELETE FROM streams WHERE id=?", (stream_id,)) | |||
def stream_insert(self, path, start, end, data, binary = False): | |||
def stream_insert(self, path, start, end, data, binary=False): | |||
"""Insert new data into the database. | |||
path: Path at which to add the data | |||
start: Starting timestamp | |||
@@ -551,8 +553,8 @@ class NilmDB(object): | |||
dbinterval.db_startpos, | |||
dbinterval.db_endpos) | |||
def stream_extract(self, path, start = None, end = None, | |||
count = False, markup = False, binary = False): | |||
def stream_extract(self, path, start=None, end=None, | |||
count=False, markup=False, binary=False): | |||
""" | |||
Returns (data, restart) tuple. | |||
@@ -632,7 +634,7 @@ class NilmDB(object): | |||
full_result = b"".join(result) | |||
return (full_result, restart) | |||
def stream_remove(self, path, start = None, end = None): | |||
def stream_remove(self, path, start=None, end=None): | |||
""" | |||
Remove data from the specified time interval within a stream. | |||
@@ -659,7 +661,7 @@ class NilmDB(object): | |||
# Can't remove intervals from within the iterator, so we need to | |||
# remember what's currently in the intersection now. | |||
all_candidates = list(intervals.intersection(to_remove, orig = True)) | |||
all_candidates = list(intervals.intersection(to_remove, orig=True)) | |||
remove_start = None | |||
remove_end = None | |||
@@ -4,16 +4,14 @@ | |||
# nilmdb.py, but will pull the nilmdb module instead. | |||
import nilmdb.server | |||
from nilmdb.utils.printf import * | |||
from nilmdb.utils.printf import sprintf | |||
from nilmdb.server.errors import NilmDBError | |||
from nilmdb.utils.time import string_to_timestamp | |||
import cherrypy | |||
import sys | |||
import os | |||
import socket | |||
import json | |||
import decorator | |||
import psutil | |||
import traceback | |||
@@ -32,10 +30,12 @@ from nilmdb.server.serverutil import ( | |||
# Add CORS_allow tool | |||
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow) | |||
class NilmApp(object): | |||
def __init__(self, db): | |||
self.db = db | |||
# CherryPy apps | |||
class Root(NilmApp): | |||
"""Root application for NILM database""" | |||
@@ -71,11 +71,14 @@ class Root(NilmApp): | |||
path = self.db.get_basepath() | |||
usage = psutil.disk_usage(path) | |||
dbsize = nilmdb.utils.du(path) | |||
return { "path": path, | |||
"size": dbsize, | |||
"other": max(usage.used - dbsize, 0), | |||
"reserved": max(usage.total - usage.used - usage.free, 0), | |||
"free": usage.free } | |||
return { | |||
"path": path, | |||
"size": dbsize, | |||
"other": max(usage.used - dbsize, 0), | |||
"reserved": max(usage.total - usage.used - usage.free, 0), | |||
"free": usage.free | |||
} | |||
class Stream(NilmApp): | |||
"""Stream-specific operations""" | |||
@@ -88,7 +91,8 @@ class Stream(NilmApp): | |||
start = string_to_timestamp(start_param) | |||
except Exception: | |||
raise cherrypy.HTTPError("400 Bad Request", sprintf( | |||
"invalid start (%s): must be a numeric timestamp", start_param)) | |||