Browse Source

Fix flake8-reported code style issues

tags/nilmdb-2.0.0
Jim Paris 1 year ago
parent
commit
2ed544bd30
45 changed files with 498 additions and 382 deletions
  1. +2
    -1
      Makefile
  2. +0
    -9
      nilmdb/__init__.py
  3. +40
    -32
      nilmdb/client/client.py
  4. +16
    -8
      nilmdb/client/errors.py
  5. +38
    -36
      nilmdb/client/httpclient.py
  6. +21
    -20
      nilmdb/client/numpyclient.py
  7. +24
    -21
      nilmdb/cmdline/cmdline.py
  8. +4
    -3
      nilmdb/cmdline/create.py
  9. +7
    -5
      nilmdb/cmdline/destroy.py
  10. +7
    -5
      nilmdb/cmdline/extract.py
  11. +4
    -5
      nilmdb/cmdline/help.py
  12. +5
    -3
      nilmdb/cmdline/info.py
  13. +9
    -5
      nilmdb/cmdline/insert.py
  14. +12
    -12
      nilmdb/cmdline/intervals.py
  15. +11
    -8
      nilmdb/cmdline/list.py
  16. +4
    -2
      nilmdb/cmdline/metadata.py
  17. +9
    -6
      nilmdb/cmdline/remove.py
  18. +4
    -3
      nilmdb/cmdline/rename.py
  19. +0
    -2
      nilmdb/fsck/__init__.py
  20. +1
    -1
      nilmdb/fsck/fsck.py
  21. +26
    -23
      nilmdb/scripts/nilmdb_server.py
  22. +2
    -0
      nilmdb/scripts/nilmtool.py
  23. +1
    -1
      nilmdb/server/__init__.py
  24. +32
    -31
      nilmdb/server/bulkdata.py
  25. +4
    -1
      nilmdb/server/errors.py
  26. +29
    -27
      nilmdb/server/nilmdb.py
  27. +63
    -51
      nilmdb/server/server.py
  28. +29
    -19
      nilmdb/server/serverutil.py
  29. +1
    -0
      nilmdb/utils/atomic.py
  30. +4
    -1
      nilmdb/utils/diskusage.py
  31. +2
    -1
      nilmdb/utils/fallocate.py
  32. +14
    -3
      nilmdb/utils/interval.py
  33. +2
    -0
      nilmdb/utils/iterator.py
  34. +2
    -0
      nilmdb/utils/lock.py
  35. +11
    -9
      nilmdb/utils/lrucache.py
  36. +7
    -5
      nilmdb/utils/mustclose.py
  37. +4
    -0
      nilmdb/utils/printf.py
  38. +6
    -6
      nilmdb/utils/serializer.py
  39. +4
    -3
      nilmdb/utils/sort.py
  40. +6
    -6
      nilmdb/utils/threadsafety.py
  41. +13
    -2
      nilmdb/utils/time.py
  42. +2
    -3
      nilmdb/utils/timer.py
  43. +11
    -3
      nilmdb/utils/timestamper.py
  44. +1
    -0
      requirements.txt
  45. +4
    -0
      setup.cfg

+ 2
- 1
Makefile View File

@@ -21,7 +21,8 @@ docs:
make -C docs

lint:
pylint3 --rcfile=.pylintrc nilmdb
flake8 nilmdb --exclude=fsck.py,nilmdb_fsck.py
ctrl: lint

test:
ifneq ($(INSIDE_EMACS),)


+ 0
- 9
nilmdb/__init__.py View File

@@ -1,14 +1,5 @@
"""Main NilmDB import"""

# These aren't imported automatically, because loading the server
# stuff isn't always necessary.
#from nilmdb.server import NilmDB, Server
#from nilmdb.client import Client

from nilmdb._version import get_versions
__version__ = get_versions()['version']
del get_versions

from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

+ 40
- 32
nilmdb/client/client.py View File

@@ -6,20 +6,21 @@ import nilmdb.utils
import nilmdb.client.httpclient
from nilmdb.client.errors import ClientError

import time
import json
import contextlib

from nilmdb.utils.time import timestamp_to_string, string_to_timestamp


def extract_timestamp(line):
"""Extract just the timestamp from a line of data text"""
return string_to_timestamp(line.split()[0])


class Client(object):
"""Main client interface to the Nilm database."""

def __init__(self, url, post_json = False):
def __init__(self, url, post_json=False):
"""Initialize client with given URL. If post_json is true,
POST requests are sent with Content-Type 'application/json'
instead of the default 'x-www-form-urlencoded'."""
@@ -38,7 +39,7 @@ class Client(object):
if self.post_json:
# If we're posting as JSON, we don't need to encode it further here
return data
return json.dumps(data, separators=(',',':'))
return json.dumps(data, separators=(',', ':'))

def close(self):
"""Close the connection; safe to call multiple times"""
@@ -57,7 +58,7 @@ class Client(object):
as a dictionary."""
return self.http.get("dbinfo")

def stream_list(self, path = None, layout = None, extended = False):
def stream_list(self, path=None, layout=None, extended=False):
"""Return a sorted list of [path, layout] lists. If 'path' or
'layout' are specified, only return streams that match those
exact values. If 'extended' is True, the returned lists have
@@ -71,11 +72,11 @@ class Client(object):
if extended:
params["extended"] = 1
streams = self.http.get("stream/list", params)
return nilmdb.utils.sort.sort_human(streams, key = lambda s: s[0])
return nilmdb.utils.sort.sort_human(streams, key=lambda s: s[0])

def stream_get_metadata(self, path, keys = None):
def stream_get_metadata(self, path, keys=None):
"""Get stream metadata"""
params = { "path": path }
params = {"path": path}
if keys is not None:
params["key"] = keys
return self.http.get("stream/get_metadata", params)
@@ -86,7 +87,7 @@ class Client(object):
params = {
"path": path,
"data": self._json_post_param(data)
}
}
return self.http.post("stream/set_metadata", params)

def stream_update_metadata(self, path, data):
@@ -94,27 +95,33 @@ class Client(object):
params = {
"path": path,
"data": self._json_post_param(data)
}
}
return self.http.post("stream/update_metadata", params)

def stream_create(self, path, layout):
"""Create a new stream"""
params = { "path": path,
"layout" : layout }
params = {
"path": path,
"layout": layout
}
return self.http.post("stream/create", params)

def stream_destroy(self, path):
"""Delete stream. Fails if any data is still present."""
params = { "path": path }
params = {
"path": path
}
return self.http.post("stream/destroy", params)

def stream_rename(self, oldpath, newpath):
"""Rename a stream."""
params = { "oldpath": oldpath,
"newpath": newpath }
params = {
"oldpath": oldpath,
"newpath": newpath
}
return self.http.post("stream/rename", params)

def stream_remove(self, path, start = None, end = None):
def stream_remove(self, path, start=None, end=None):
"""Remove data from the specified time range"""
params = {
"path": path
@@ -129,7 +136,7 @@ class Client(object):
return total

@contextlib.contextmanager
def stream_insert_context(self, path, start = None, end = None):
def stream_insert_context(self, path, start=None, end=None):
"""Return a context manager that allows data to be efficiently
inserted into a stream in a piecewise manner. Data is
provided as ASCII lines, and is aggregated and sent to the
@@ -152,7 +159,7 @@ class Client(object):
ctx.finalize()
ctx.destroy()

def stream_insert(self, path, data, start = None, end = None):
def stream_insert(self, path, data, start=None, end=None):
"""Insert rows of data into a stream. data should be a string
or iterable that provides ASCII data that matches the database
layout for path. Data is passed through stream_insert_context,
@@ -166,7 +173,7 @@ class Client(object):
ctx.insert(chunk)
return ctx.last_response

def stream_insert_block(self, path, data, start, end, binary = False):
def stream_insert_block(self, path, data, start, end, binary=False):
"""Insert a single fixed block of data into the stream. It is
sent directly to the server in one block with no further
processing.
@@ -183,7 +190,7 @@ class Client(object):
params["binary"] = 1
return self.http.put("stream/insert", data, params)

def stream_intervals(self, path, start = None, end = None, diffpath = None):
def stream_intervals(self, path, start=None, end=None, diffpath=None):
"""
Return a generator that yields each stream interval.

@@ -201,8 +208,8 @@ class Client(object):
params["end"] = timestamp_to_string(end)
return self.http.get_gen("stream/intervals", params)

def stream_extract(self, path, start = None, end = None,
count = False, markup = False, binary = False):
def stream_extract(self, path, start=None, end=None,
count=False, markup=False, binary=False):
"""
Extract data from a stream. Returns a generator that yields
lines of ASCII-formatted data that matches the database
@@ -232,16 +239,17 @@ class Client(object):
params["markup"] = 1
if binary:
params["binary"] = 1
return self.http.get_gen("stream/extract", params, binary = binary)
return self.http.get_gen("stream/extract", params, binary=binary)

def stream_count(self, path, start = None, end = None):
def stream_count(self, path, start=None, end=None):
"""
Return the number of rows of data in the stream that satisfy
the given timestamps.
"""
counts = list(self.stream_extract(path, start, end, count = True))
counts = list(self.stream_extract(path, start, end, count=True))
return int(counts[0])


class StreamInserter(object):
"""Object returned by stream_insert_context() that manages
the insertion of rows of data into a particular path.
@@ -330,7 +338,7 @@ class StreamInserter(object):

# Send the block once we have enough data
if self._block_len >= maxdata:
self._send_block(final = False)
self._send_block(final=False)
if self._block_len >= self._max_data_after_send:
raise ValueError("too much data left over after trying"
" to send intermediate block; is it"
@@ -357,12 +365,12 @@ class StreamInserter(object):

If more data is inserted after a finalize(), it will become
part of a new interval and there may be a gap left in-between."""
self._send_block(final = True)
self._send_block(final=True)

def send(self):
"""Send any data that we might have buffered up. Does not affect
any other treatment of timestamps or endpoints."""
self._send_block(final = False)
self._send_block(final=False)

def _get_first_noncomment(self, block):
"""Return the (start, end) indices of the first full line in
@@ -392,7 +400,7 @@ class StreamInserter(object):
raise IndexError
end = start

def _send_block(self, final = False):
def _send_block(self, final=False):
"""Send data currently in the block. The data sent will
consist of full lines only, so some might be left over."""
# Build the full string to send
@@ -405,7 +413,7 @@ class StreamInserter(object):
(spos, epos) = self._get_first_noncomment(block)
start_ts = extract_timestamp(block[spos:epos])
except (ValueError, IndexError):
pass # no timestamp is OK, if we have no data
pass # no timestamp is OK, if we have no data

if final:
# For a final block, it must end in a newline, and the
@@ -420,7 +428,7 @@ class StreamInserter(object):
end_ts = extract_timestamp(block[spos:epos])
end_ts += nilmdb.utils.time.epsilon
except (ValueError, IndexError):
pass # no timestamp is OK, if we have no data
pass # no timestamp is OK, if we have no data
self._block_data = []
self._block_len = 0

@@ -447,7 +455,7 @@ class StreamInserter(object):
# the server complain so that the error is the same
# as if we hadn't done this chunking.
end_ts = self._interval_end
self._block_data = [ block[spos:] ]
self._block_data = [block[spos:]]
self._block_len = (epos - spos)
block = block[:spos]

@@ -465,6 +473,6 @@ class StreamInserter(object):

# Send it
self.last_response = self._client.stream_insert_block(
self._path, block, start_ts, end_ts, binary = False)
self._path, block, start_ts, end_ts, binary=False)

return

+ 16
- 8
nilmdb/client/errors.py View File

@@ -1,19 +1,21 @@
"""HTTP client errors"""

from nilmdb.utils.printf import *
from nilmdb.utils.printf import sprintf


class Error(Exception):
"""Base exception for both ClientError and ServerError responses"""
def __init__(self,
status = "Unspecified error",
message = None,
url = None,
traceback = None):
status="Unspecified error",
message=None,
url=None,
traceback=None):
super().__init__(status)
self.status = status # e.g. "400 Bad Request"
self.message = message # textual message from the server
self.url = url # URL we were requesting
self.traceback = traceback # server traceback, if available
self.traceback = traceback # server traceback, if available

def _format_error(self, show_url):
s = sprintf("[%s]", self.status)
if self.message:
@@ -23,11 +25,17 @@ class Error(Exception):
if self.traceback:
s += sprintf("\nServer traceback:\n%s", self.traceback)
return s

def __str__(self):
return self._format_error(show_url = False)
return self._format_error(show_url=False)

def __repr__(self):
return self._format_error(show_url = True)
return self._format_error(show_url=True)


class ClientError(Error):
pass


class ServerError(Error):
pass

+ 38
- 36
nilmdb/client/httpclient.py View File

@@ -1,15 +1,15 @@
"""HTTP client library"""

import nilmdb.utils
from nilmdb.client.errors import ClientError, ServerError, Error

import json
import urllib.parse
import requests


class HTTPClient(object):
"""Class to manage and perform HTTP requests from the client"""
def __init__(self, baseurl = "", post_json = False, verify_ssl = True):
def __init__(self, baseurl="", post_json=False, verify_ssl=True):
"""If baseurl is supplied, all other functions that take
a URL can be given a relative URL instead."""
# Verify / clean up URL
@@ -32,10 +32,12 @@ class HTTPClient(object):
# Default variables for exception. We use the entire body as
# the default message, in case we can't extract it from a JSON
# response.
args = { "url" : url,
"status" : str(code),
"message" : body,
"traceback" : None }
args = {
"url": url,
"status": str(code),
"message": body,
"traceback": None
}
try:
# Fill with server-provided data if we can
jsonerror = json.loads(body)
@@ -49,8 +51,8 @@ class HTTPClient(object):
else:
if code >= 500 and code <= 599:
if args["message"] is None:
args["message"] = ("(no message; try disabling " +
"response.stream option in " +
args["message"] = ("(no message; try disabling "
"response.stream option in "
"nilmdb.server for better debugging)")
raise ServerError(**args)
else:
@@ -74,11 +76,11 @@ class HTTPClient(object):
headers = {}
headers["Connection"] = "close"
response = session.request(method, url,
params = query_data,
data = body_data,
stream = stream,
headers = headers,
verify = self.verify_ssl)
params=query_data,
data=body_data,
stream=stream,
headers=headers,
verify=self.verify_ssl)

# Close the connection. If it's a generator (stream =
# True), the requests library shouldn't actually close the
@@ -86,8 +88,8 @@ class HTTPClient(object):
# response.
session.close()
except requests.RequestException as e:
raise ServerError(status = "502 Error", url = url,
message = str(e))
raise ServerError(status="502 Error", url=url,
message=str(e))
if response.status_code != 200:
self._handle_error(url, response.status_code, response.content)
self._last_response = response
@@ -98,46 +100,46 @@ class HTTPClient(object):
return (response, False)

# Normal versions that return data directly
def _req(self, method, url, query = None, body = None, headers = None):
def _req(self, method, url, query=None, body=None, headers=None):
"""
Make a request and return the body data as a string or parsed
JSON object, or raise an error if it contained an error.
"""
(response, isjson) = self._do_req(method, url, query, body,
stream = False, headers = headers)
stream=False, headers=headers)
if isjson:
return json.loads(response.content)
return response.content

def get(self, url, params = None):
def get(self, url, params=None):
"""Simple GET (parameters in URL)"""
return self._req("GET", url, params, None)

def post(self, url, params = None):
def post(self, url, params=None):
"""Simple POST (parameters in body)"""
if self.post_json:
return self._req("POST", url, None,
json.dumps(params),
{ 'Content-type': 'application/json' })
{'Content-type': 'application/json'})
else:
return self._req("POST", url, None, params)

def put(self, url, data, params = None,
content_type = "application/octet-stream"):
def put(self, url, data, params=None,
content_type="application/octet-stream"):
"""Simple PUT (parameters in URL, data in body)"""
h = { 'Content-type': content_type }
return self._req("PUT", url, query = params, body = data, headers = h)
h = {'Content-type': content_type}
return self._req("PUT", url, query=params, body=data, headers=h)

# Generator versions that return data one line at a time.
def _req_gen(self, method, url, query = None, body = None,
headers = None, binary = False):
def _req_gen(self, method, url, query=None, body=None,
headers=None, binary=False):
"""
Make a request and return a generator that gives back strings
or JSON decoded lines of the body data, or raise an error if
it contained an eror.
"""
(response, isjson) = self._do_req(method, url, query, body,
stream = True, headers = headers)
stream=True, headers=headers)

# Like the iter_lines function in Requests, but only splits on
# the specified line ending.
@@ -159,27 +161,27 @@ class HTTPClient(object):

# Yield the chunks or lines as requested
if binary:
for chunk in response.iter_content(chunk_size = 65536):
for chunk in response.iter_content(chunk_size=65536):
yield chunk
elif isjson:
for line in lines(response.iter_content(chunk_size = 1),
ending = b'\r\n'):
for line in lines(response.iter_content(chunk_size=1),
ending=b'\r\n'):
yield json.loads(line)
else:
for line in lines(response.iter_content(chunk_size = 65536),
ending = b'\n'):
for line in lines(response.iter_content(chunk_size=65536),
ending=b'\n'):
yield line

def get_gen(self, url, params = None, binary = False):
def get_gen(self, url, params=None, binary=False):
"""Simple GET (parameters in URL) returning a generator"""
return self._req_gen("GET", url, params, binary = binary)
return self._req_gen("GET", url, params, binary=binary)

def post_gen(self, url, params = None):
def post_gen(self, url, params=None):
"""Simple POST (parameters in body) returning a generator"""
if self.post_json:
return self._req_gen("POST", url, None,
json.dumps(params),
{ 'Content-type': 'application/json' })
{'Content-type': 'application/json'})
else:
return self._req_gen("POST", url, None, params)



+ 21
- 20
nilmdb/client/numpyclient.py View File

@@ -9,10 +9,9 @@ import nilmdb.client.httpclient
from nilmdb.client.errors import ClientError

import contextlib
from nilmdb.utils.time import timestamp_to_string, string_to_timestamp

import numpy
import io

def layout_to_dtype(layout):
ltype = layout.split('_')[0]
@@ -31,6 +30,7 @@ def layout_to_dtype(layout):
dtype = [('timestamp', '<i8'), ('data', atype, lcount)]
return numpy.dtype(dtype)


class NumpyClient(nilmdb.client.client.Client):
"""Subclass of nilmdb.client.Client that adds additional methods for
extracting and inserting data via Numpy arrays."""
@@ -43,9 +43,9 @@ class NumpyClient(nilmdb.client.client.Client):
layout = streams[0][1]
return layout_to_dtype(layout)

def stream_extract_numpy(self, path, start = None, end = None,
layout = None, maxrows = 100000,
structured = False):
def stream_extract_numpy(self, path, start=None, end=None,
layout=None, maxrows=100000,
structured=False):
"""
Extract data from a stream. Returns a generator that yields
Numpy arrays of up to 'maxrows' of data each.
@@ -67,7 +67,7 @@ class NumpyClient(nilmdb.client.client.Client):
chunks = []
total_len = 0
maxsize = dtype.itemsize * maxrows
for data in self.stream_extract(path, start, end, binary = True):
for data in self.stream_extract(path, start, end, binary=True):
# Add this block of binary data
chunks.append(data)
total_len += len(data)
@@ -76,7 +76,7 @@ class NumpyClient(nilmdb.client.client.Client):
while total_len >= maxsize:
assembled = b"".join(chunks)
total_len -= maxsize
chunks = [ assembled[maxsize:] ]
chunks = [assembled[maxsize:]]
block = assembled[:maxsize]
yield to_numpy(block)

@@ -84,8 +84,8 @@ class NumpyClient(nilmdb.client.client.Client):
yield to_numpy(b"".join(chunks))

@contextlib.contextmanager
def stream_insert_numpy_context(self, path, start = None, end = None,
layout = None):
def stream_insert_numpy_context(self, path, start=None, end=None,
layout=None):
"""Return a context manager that allows data to be efficiently
inserted into a stream in a piecewise manner. Data is
provided as Numpy arrays, and is aggregated and sent to the
@@ -104,8 +104,8 @@ class NumpyClient(nilmdb.client.client.Client):
ctx.finalize()
ctx.destroy()

def stream_insert_numpy(self, path, data, start = None, end = None,
layout = None):
def stream_insert_numpy(self, path, data, start=None, end=None,
layout=None):
"""Insert data into a stream. data should be a Numpy array
which will be passed through stream_insert_numpy_context to
break it into chunks etc. See the help for that function
@@ -118,6 +118,7 @@ class NumpyClient(nilmdb.client.client.Client):
ctx.insert(chunk)
return ctx.last_response


class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
"""Object returned by stream_insert_numpy_context() that manages
the insertion of rows of data into a particular path.
@@ -160,9 +161,9 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
# Convert to structured array
sarray = numpy.zeros(array.shape[0], dtype=self._dtype)
try:
sarray['timestamp'] = array[:,0]
sarray['timestamp'] = array[:, 0]
# Need the squeeze in case sarray['data'] is 1 dimensional
sarray['data'] = numpy.squeeze(array[:,1:])
sarray['data'] = numpy.squeeze(array[:, 1:])
except (IndexError, ValueError):
raise ValueError("wrong number of fields for this data type")
array = sarray
@@ -188,15 +189,15 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):

# Send if it's too long
if self._block_rows >= maxrows:
self._send_block(final = False)
self._send_block(final=False)

def _send_block(self, final = False):
def _send_block(self, final=False):
"""Send the data current stored up. One row might be left
over if we need its timestamp saved."""

# Build the full array to send
if self._block_rows == 0:
array = numpy.zeros(0, dtype = self._dtype)
array = numpy.zeros(0, dtype=self._dtype)
else:
array = numpy.hstack(self._block_arrays)

@@ -207,7 +208,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
try:
start_ts = array['timestamp'][0]
except IndexError:
pass # no timestamp is OK, if we have no data
pass # no timestamp is OK, if we have no data

# Get ending timestamp
if final:
@@ -220,7 +221,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
end_ts = array['timestamp'][-1]
end_ts += nilmdb.utils.time.epsilon
except IndexError:
pass # no timestamp is OK, if we have no data
pass # no timestamp is OK, if we have no data
self._block_arrays = []
self._block_rows = 0

@@ -240,7 +241,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
# the server complain so that the error is the same
# as if we hadn't done this chunking.
end_ts = self._interval_end
self._block_arrays = [ array[-1:] ]
self._block_arrays = [array[-1:]]
self._block_rows = 1
array = array[:-1]

@@ -257,6 +258,6 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
# Send it
data = array.tostring()
self.last_response = self._client.stream_insert_block(
self._path, data, start_ts, end_ts, binary = True)
self._path, data, start_ts, end_ts, binary=True)

return

+ 24
- 21
nilmdb/cmdline/cmdline.py View File

@@ -2,7 +2,7 @@

import nilmdb.client

from nilmdb.utils.printf import *
from nilmdb.utils.printf import fprintf, sprintf
import datetime_tz
import nilmdb.utils.time

@@ -16,13 +16,14 @@ import argcomplete

# Valid subcommands. Defined in separate files just to break
# things up -- they're still called with Cmdline as self.
subcommands = [ "help", "info", "create", "rename", "list", "intervals",
"metadata", "insert", "extract", "remove", "destroy" ]
subcommands = ["help", "info", "create", "rename", "list", "intervals",
"metadata", "insert", "extract", "remove", "destroy"]

# Import the subcommand modules
subcmd_mods = {}
for cmd in subcommands:
subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist = [ cmd ])
subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist=[cmd])


class JimArgumentParser(argparse.ArgumentParser):
def parse_args(self, args=None, namespace=None):
@@ -30,18 +31,19 @@ class JimArgumentParser(argparse.ArgumentParser):
# --version". This makes "nilmtool cmd --version" work, which
# is needed by help2man.
if "--version" in (args or sys.argv[1:]):
args = [ "--version" ]
args = ["--version"]
return argparse.ArgumentParser.parse_args(self, args, namespace)

def error(self, message):
self.print_usage(sys.stderr)
self.exit(2, sprintf("error: %s\n", message))


class Complete(object):
# Completion helpers, for using argcomplete (see
# extras/nilmtool-bash-completion.sh)
def escape(self, s):
quote_chars = [ "\\", "\"", "'", " " ]
quote_chars = ["\\", "\"", "'", " "]
for char in quote_chars:
s = s.replace(char, "\\" + char)
return s
@@ -54,18 +56,18 @@ class Complete(object):

def path(self, prefix, parsed_args, **kwargs):
client = nilmdb.client.Client(parsed_args.url)
return ( self.escape(s[0])
for s in client.stream_list()
if s[0].startswith(prefix) )
return (self.escape(s[0])
for s in client.stream_list()
if s[0].startswith(prefix))

def layout(self, prefix, parsed_args, **kwargs):
types = [ "int8", "int16", "int32", "int64",
"uint8", "uint16", "uint32", "uint64",
"float32", "float64" ]
types = ["int8", "int16", "int32", "int64",
"uint8", "uint16", "uint32", "uint64",
"float32", "float64"]
layouts = []
for i in range(1,10):
for i in range(1, 10):
layouts.extend([(t + "_" + str(i)) for t in types])
return ( l for l in layouts if l.startswith(prefix) )
return (l for l in layouts if l.startswith(prefix))

def meta_key(self, prefix, parsed_args, **kwargs):
return (kv.split('=')[0] for kv
@@ -77,21 +79,22 @@ class Complete(object):
if not path:
return []
results = []
for (k,v) in client.stream_get_metadata(path).items():
for (k, v) in client.stream_get_metadata(path).items():
kv = self.escape(k + '=' + v)
if kv.startswith(prefix):
results.append(kv)
return results


class Cmdline(object):

def __init__(self, argv = None):
def __init__(self, argv=None):
self.argv = argv or sys.argv[1:]
self.client = None
self.def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
self.subcmd = {}
self.complete = Complete()
self.complete_output_stream = None # overridden by test suite
self.complete_output_stream = None # overridden by test suite

def arg_time(self, toparse):
"""Parse a time string argument"""
@@ -103,14 +106,14 @@ class Cmdline(object):

# Set up the parser
def parser_setup(self):
self.parser = JimArgumentParser(add_help = False,
formatter_class = def_form)
self.parser = JimArgumentParser(add_help=False,
formatter_class=def_form)

group = self.parser.add_argument_group("General options")
group.add_argument("-h", "--help", action='help',
help='show this help message and exit')
group.add_argument("-v", "--version", action="version",
version = nilmdb.__version__)
version=nilmdb.__version__)

group = self.parser.add_argument_group("Server")
group.add_argument("-u", "--url", action="store",
@@ -158,7 +161,7 @@ class Cmdline(object):
# unless the particular command requests that we don't.
if "no_test_connect" not in self.args:
try:
server_version = self.client.version()
self.client.version()
except nilmdb.client.Error as e:
self.die("error connecting to server: %s", str(e))



+ 4
- 3
nilmdb/cmdline/create.py View File

@@ -1,11 +1,11 @@
from nilmdb.utils.printf import *
import nilmdb.client

from argparse import RawDescriptionHelpFormatter as raw_form


def setup(self, sub):
cmd = sub.add_parser("create", help="Create a new stream",
formatter_class = raw_form,
formatter_class=raw_form,
description="""
Create a new empty stream at the specified path and with the specified
layout type.
@@ -19,7 +19,7 @@ Layout types are of the format: type_count
For example, 'float32_8' means the data for this stream has 8 columns of
32-bit floating point values.
""")
cmd.set_defaults(handler = cmd_create)
cmd.set_defaults(handler=cmd_create)
group = cmd.add_argument_group("Required arguments")
group.add_argument("path",
help="Path (in database) of new stream, e.g. /foo/bar",
@@ -29,6 +29,7 @@ Layout types are of the format: type_count
).completer = self.complete.layout
return cmd


def cmd_create(self):
"""Create new stream"""
try:


+ 7
- 5
nilmdb/cmdline/destroy.py View File

@@ -1,12 +1,13 @@
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
import nilmdb.client
import fnmatch

from argparse import ArgumentDefaultsHelpFormatter as def_form


def setup(self, sub):
cmd = sub.add_parser("destroy", help="Delete a stream and all data",
formatter_class = def_form,
formatter_class=def_form,
description="""
Destroy the stream at the specified path.
The stream must be empty. All metadata
@@ -14,7 +15,7 @@ def setup(self, sub):

Wildcards and multiple paths are supported.
""")
cmd.set_defaults(handler = cmd_destroy)
cmd.set_defaults(handler=cmd_destroy)
group = cmd.add_argument_group("Options")
group.add_argument("-R", "--remove", action="store_true",
help="Remove all data before destroying stream")
@@ -27,9 +28,10 @@ def setup(self, sub):
).completer = self.complete.path
return cmd


def cmd_destroy(self):
"""Destroy stream"""
streams = [ s[0] for s in self.client.stream_list() ]
streams = [s[0] for s in self.client.stream_list()]
paths = []
for path in self.args.path:
new = fnmatch.filter(streams, path)
@@ -43,7 +45,7 @@ def cmd_destroy(self):

try:
if self.args.remove:
count = self.client.stream_remove(path)
self.client.stream_remove(path)
self.client.stream_destroy(path)
except nilmdb.client.ClientError as e:
self.die("error destroying stream: %s", str(e))

+ 7
- 5
nilmdb/cmdline/extract.py View File

@@ -1,15 +1,15 @@

from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
import nilmdb.client
import sys


def setup(self, sub):
cmd = sub.add_parser("extract", help="Extract data",
description="""
Extract data from a stream.
""")
cmd.set_defaults(verify = cmd_extract_verify,
handler = cmd_extract)
cmd.set_defaults(verify=cmd_extract_verify,
handler=cmd_extract)

group = cmd.add_argument_group("Data selection")
group.add_argument("path",
@@ -40,15 +40,17 @@ def setup(self, sub):
help="Just output a count of matched data points")
return cmd


def cmd_extract_verify(self):
if self.args.start > self.args.end:
self.parser.error("start is after end")

if self.args.binary:
if (self.args.bare or self.args.annotate or self.args.markup or
self.args.timestamp_raw or self.args.count):
self.args.timestamp_raw or self.args.count):
self.parser.error("--binary cannot be combined with other options")


def cmd_extract(self):
streams = self.client.stream_list(self.args.path)
if len(streams) != 1:


+ 4
- 5
nilmdb/cmdline/help.py View File

@@ -1,7 +1,5 @@
from nilmdb.utils.printf import *

import argparse
import sys


def setup(self, sub):
cmd = sub.add_parser("help", help="Show detailed help for a command",
@@ -9,14 +7,15 @@ def setup(self, sub):
Show help for a command. 'help command' is
the same as 'command --help'.
""")
cmd.set_defaults(handler = cmd_help)
cmd.set_defaults(no_test_connect = True)
cmd.set_defaults(handler=cmd_help)
cmd.set_defaults(no_test_connect=True)
cmd.add_argument("command", nargs="?",
help="Command to get help about")
cmd.add_argument("rest", nargs=argparse.REMAINDER,
help=argparse.SUPPRESS)
return cmd


def cmd_help(self):
if self.args.command in self.subcmd:
self.subcmd[self.args.command].print_help()


+ 5
- 3
nilmdb/cmdline/info.py View File

@@ -1,19 +1,21 @@
import nilmdb.client
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
from nilmdb.utils import human_size

from argparse import ArgumentDefaultsHelpFormatter as def_form


def setup(self, sub):
cmd = sub.add_parser("info", help="Server information",
formatter_class = def_form,
formatter_class=def_form,
description="""
List information about the server, like
version.
""")
cmd.set_defaults(handler = cmd_info)
cmd.set_defaults(handler=cmd_info)
return cmd


def cmd_info(self):
"""Print info about the server"""
printf("Client version: %s\n", nilmdb.__version__)


+ 9
- 5
nilmdb/cmdline/insert.py View File

@@ -1,17 +1,18 @@
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
import nilmdb.client
import nilmdb.utils.timestamper as timestamper
import nilmdb.utils.time

import sys


def setup(self, sub):
cmd = sub.add_parser("insert", help="Insert data",
description="""
Insert data into a stream.
""")
cmd.set_defaults(verify = cmd_insert_verify,
handler = cmd_insert)
cmd.set_defaults(verify=cmd_insert_verify,
handler=cmd_insert)
cmd.add_argument("-q", "--quiet", action='store_true',
help='suppress unnecessary messages')

@@ -61,21 +62,24 @@ def setup(self, sub):
group.add_argument("path",
help="Path of stream, e.g. /foo/bar",
).completer = self.complete.path
group.add_argument("file", nargs = '?', default='-',
group.add_argument("file", nargs='?', default='-',
help="File to insert (default: - (stdin))")
return cmd


def cmd_insert_verify(self):
if self.args.timestamp:
if not self.args.rate:
self.die("error: --rate is needed, but was not specified")
if not self.args.filename and self.args.start is None:
self.die("error: need --start or --filename when adding timestamps")
self.die("error: need --start or --filename "
"when adding timestamps")
else:
if self.args.start is None or self.args.end is None:
self.die("error: when not adding timestamps, --start and "
"--end are required")


def cmd_insert(self):
# Find requested stream
streams = self.client.stream_list(self.args.path)


+ 12
- 12
nilmdb/cmdline/intervals.py View File

@@ -1,14 +1,13 @@
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
import nilmdb.utils.time
from nilmdb.utils.interval import Interval

import fnmatch
import argparse
from argparse import ArgumentDefaultsHelpFormatter as def_form


def setup(self, sub):
cmd = sub.add_parser("intervals", help="List intervals",
formatter_class = def_form,
formatter_class=def_form,
description="""
List intervals in a stream, similar to
'list --detail path'.
@@ -17,8 +16,8 @@ def setup(self, sub):
interval ranges that are present in 'path'
and not present in 'diffpath' are printed.
""")
cmd.set_defaults(verify = cmd_intervals_verify,
handler = cmd_intervals)
cmd.set_defaults(verify=cmd_intervals_verify,
handler=cmd_intervals)

group = cmd.add_argument_group("Stream selection")
group.add_argument("path", metavar="PATH",
@@ -48,11 +47,13 @@ def setup(self, sub):

return cmd


def cmd_intervals_verify(self):
if self.args.start is not None and self.args.end is not None:
if self.args.start >= self.args.end:
self.parser.error("start must precede end")


def cmd_intervals(self):
"""List intervals in a stream"""
if self.args.timestamp_raw:
@@ -61,11 +62,11 @@ def cmd_intervals(self):
time_string = nilmdb.utils.time.timestamp_to_human

try:
intervals = ( Interval(start, end) for (start, end) in
self.client.stream_intervals(self.args.path,
self.args.start,
self.args.end,
self.args.diff) )
intervals = (Interval(start, end) for (start, end) in
self.client.stream_intervals(self.args.path,
self.args.start,
self.args.end,
self.args.diff))
if self.args.optimize:
intervals = nilmdb.utils.interval.optimize(intervals)
for i in intervals:
@@ -73,4 +74,3 @@ def cmd_intervals(self):

except nilmdb.client.ClientError as e:
self.die("error listing intervals: %s", str(e))


+ 11
- 8
nilmdb/cmdline/list.py View File

@@ -1,21 +1,21 @@
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
import nilmdb.utils.time

import fnmatch
import argparse
from argparse import ArgumentDefaultsHelpFormatter as def_form


def setup(self, sub):
cmd = sub.add_parser("list", help="List streams",
formatter_class = def_form,
formatter_class=def_form,
description="""
List streams available in the database,
optionally filtering by path. Wildcards
are accepted; non-matching paths or wildcards
are ignored.
""")
cmd.set_defaults(verify = cmd_list_verify,
handler = cmd_list)
cmd.set_defaults(verify=cmd_list_verify,
handler=cmd_list)

group = cmd.add_argument_group("Stream filtering")
group.add_argument("path", metavar="PATH", default=["*"], nargs='*',
@@ -50,6 +50,7 @@ def setup(self, sub):

return cmd


def cmd_list_verify(self):
if self.args.start is not None and self.args.end is not None:
if self.args.start >= self.args.end:
@@ -57,11 +58,13 @@ def cmd_list_verify(self):

if self.args.start is not None or self.args.end is not None:
if not self.args.detail:
self.parser.error("--start and --end only make sense with --detail")
self.parser.error("--start and --end only make sense "
"with --detail")


def cmd_list(self):
"""List available streams"""
streams = self.client.stream_list(extended = True)
streams = self.client.stream_list(extended=True)

if self.args.timestamp_raw:
time_string = nilmdb.utils.time.timestamp_to_string
@@ -94,7 +97,7 @@ def cmd_list(self):
if self.args.detail:
printed = False
for (start, end) in self.client.stream_intervals(
path, self.args.start, self.args.end):
path, self.args.start, self.args.end):
printf(" [ %s -> %s ]\n",
time_string(start), time_string(end))
printed = True


+ 4
- 2
nilmdb/cmdline/metadata.py View File

@@ -1,7 +1,8 @@
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
import nilmdb
import nilmdb.client


def setup(self, sub):
cmd = sub.add_parser("metadata", help="Get or set stream metadata",
description="""
@@ -11,7 +12,7 @@ def setup(self, sub):
usage="%(prog)s path [-g [key ...] | "
"-s key=value [...] | -u key=value [...]] | "
"-d [key ...]")
cmd.set_defaults(handler = cmd_metadata)
cmd.set_defaults(handler=cmd_metadata)

group = cmd.add_argument_group("Required arguments")
group.add_argument("path",
@@ -36,6 +37,7 @@ def setup(self, sub):
).completer = self.complete.meta_key
return cmd


def cmd_metadata(self):
"""Manipulate metadata"""
if self.args.set is not None or self.args.update is not None:


+ 9
- 6
nilmdb/cmdline/remove.py View File

@@ -1,15 +1,17 @@
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
import nilmdb.client
import fnmatch


def setup(self, sub):
cmd = sub.add_parser("remove", help="Remove data",
description="""
Remove all data from a specified time range within a
stream. If multiple streams or wildcards are provided,
the same time range is removed from all streams.
stream. If multiple streams or wildcards are
provided, the same time range is removed from all
streams.
""")
cmd.set_defaults(handler = cmd_remove)
cmd.set_defaults(handler=cmd_remove)

group = cmd.add_argument_group("Data selection")
group.add_argument("path", nargs='+',
@@ -32,8 +34,9 @@ def setup(self, sub):
help="Output number of data points removed")
return cmd


def cmd_remove(self):
streams = [ s[0] for s in self.client.stream_list() ]
streams = [s[0] for s in self.client.stream_list()]
paths = []
for path in self.args.path:
new = fnmatch.filter(streams, path)
@@ -48,7 +51,7 @@ def cmd_remove(self):
count = self.client.stream_remove(path,
self.args.start, self.args.end)
if self.args.count:
printf("%d\n", count);
printf("%d\n", count)
except nilmdb.client.ClientError as e:
self.die("error removing data: %s", str(e))



+ 4
- 3
nilmdb/cmdline/rename.py View File

@@ -1,18 +1,18 @@
from nilmdb.utils.printf import *
import nilmdb.client

from argparse import ArgumentDefaultsHelpFormatter as def_form


def setup(self, sub):
cmd = sub.add_parser("rename", help="Rename a stream",
formatter_class = def_form,
formatter_class=def_form,
description="""
Rename a stream.

Only the stream's path is renamed; no
metadata is changed.
""")
cmd.set_defaults(handler = cmd_rename)
cmd.set_defaults(handler=cmd_rename)
group = cmd.add_argument_group("Required arguments")
group.add_argument("oldpath",
help="Old path, e.g. /foo/old",
@@ -23,6 +23,7 @@ def setup(self, sub):

return cmd


def cmd_rename(self):
"""Rename a stream"""
try:


+ 0
- 2
nilmdb/fsck/__init__.py View File

@@ -1,5 +1,3 @@
"""nilmdb.fsck"""



from nilmdb.fsck.fsck import Fsck

+ 1
- 1
nilmdb/fsck/fsck.py View File

@@ -12,7 +12,7 @@ import nilmdb.server
import nilmdb.client.numpyclient
from nilmdb.utils.interval import IntervalError
from nilmdb.server.interval import Interval, IntervalSet
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf, fprintf, sprintf
from nilmdb.utils.time import timestamp_to_string

from collections import defaultdict


+ 26
- 23
nilmdb/scripts/nilmdb_server.py View File

@@ -7,34 +7,35 @@ import socket
import cherrypy
import sys


def main():
"""Main entry point for the 'nilmdb-server' command line script"""

parser = argparse.ArgumentParser(
description = 'Run the NilmDB server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
description='Run the NilmDB server',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)

parser.add_argument("-v", "--version", action="version",
version = nilmdb.__version__)
version=nilmdb.__version__)

group = parser.add_argument_group("Standard options")
group.add_argument('-a', '--address',
help = 'Only listen on the given address',
default = '0.0.0.0')
group.add_argument('-p', '--port', help = 'Listen on the given port',
type = int, default = 12380)
group.add_argument('-d', '--database', help = 'Database directory',
default = "./db")
group.add_argument('-q', '--quiet', help = 'Silence output',
action = 'store_true')
help='Only listen on the given address',
default='0.0.0.0')
group.add_argument('-p', '--port', help='Listen on the given port',
type=int, default=12380)
group.add_argument('-d', '--database', help='Database directory',
default="./db")
group.add_argument('-q', '--quiet', help='Silence output',
action='store_true')
group.add_argument('-t', '--traceback',
help = 'Provide tracebacks in client errors',
action = 'store_true', default = False)
help='Provide tracebacks in client errors',
action='store_true', default=False)

group = parser.add_argument_group("Debug options")
group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and '
group.add_argument('-y', '--yappi', help='Run under yappi profiler and '
'invoke interactive shell afterwards',
action = 'store_true')
action='store_true')

args = parser.parse_args()

@@ -45,10 +46,11 @@ def main():
# Configure the server
if not args.quiet:
cherrypy._cpconfig.environments['embedded']['log.screen'] = True

server = nilmdb.server.Server(db,
host = args.address,
port = args.port,
force_traceback = args.traceback)
host=args.address,
port=args.port,
force_traceback=args.traceback)

# Print info
if not args.quiet:
@@ -58,7 +60,7 @@ def main():
host = socket.getfqdn()
else:
host = args.address
print("Server URL: http://%s:%d/" % ( host, args.port))
print("Server URL: http://%s:%d/" % (host, args.port))
print("----")

# Run it
@@ -68,18 +70,18 @@ def main():
try:
import yappi
yappi.start()
server.start(blocking = True)
server.start(blocking=True)
finally:
yappi.stop()
stats = yappi.get_func_stats()
stats.sort("ttot")
stats.print_all()
from IPython import embed
embed(header = "Use the `yappi` or `stats` object to explore "
embed(header="Use the `yappi` or `stats` object to explore "
"further, quit to exit")
else:
server.start(blocking = True)
except nilmdb.server.serverutil.CherryPyExit as e:
server.start(blocking=True)
except nilmdb.server.serverutil.CherryPyExit:
print("Exiting due to CherryPy error", file=sys.stderr)
raise
finally:
@@ -87,5 +89,6 @@ def main():
print("Closing database")
db.close()


if __name__ == "__main__":
main()

+ 2
- 0
nilmdb/scripts/nilmtool.py View File

@@ -2,9 +2,11 @@

import nilmdb.cmdline


def main():
"""Main entry point for the 'nilmtool' command line script"""
nilmdb.cmdline.Cmdline().run()


if __name__ == "__main__":
main()

+ 1
- 1
nilmdb/server/__init__.py View File

@@ -2,7 +2,7 @@

# Set up pyximport to automatically rebuild Cython modules if needed.
import pyximport
pyximport.install(inplace = True, build_in_temp = False)
pyximport.install(inplace=True, build_in_temp=False)

from nilmdb.server.nilmdb import NilmDB
from nilmdb.server.server import Server, wsgi_application


+ 32
- 31
nilmdb/server/bulkdata.py View File

@@ -4,8 +4,8 @@
# nilmdb.py, but will pull the parent nilmdb module instead.


from nilmdb.utils.printf import *
from nilmdb.utils.time import timestamp_to_string as timestamp_to_string
from nilmdb.utils.printf import sprintf
from nilmdb.utils.time import timestamp_to_string
import nilmdb.utils

import os
@@ -22,7 +22,8 @@ from . import rocket
table_cache_size = 32
fd_cache_size = 8

@nilmdb.utils.must_close(wrap_verify = False)

@nilmdb.utils.must_close(wrap_verify=False)
class BulkData(object):
def __init__(self, basepath, **kwargs):
if isinstance(basepath, str):
@@ -103,7 +104,7 @@ class BulkData(object):

if path[0:1] != b'/':
raise ValueError("paths must start with / ")
[ group, node ] = path.rsplit(b"/", 1)
[group, node] = path.rsplit(b"/", 1)
if group == b'':
raise ValueError("invalid path; path must contain at least one "
"folder")
@@ -129,7 +130,7 @@ class BulkData(object):
if not os.path.isdir(ospath):
os.mkdir(ospath)
made_dirs.append(ospath)
except Exception as e:
except Exception:
# Remove paths that we created
for ospath in reversed(made_dirs):
os.rmdir(ospath)
@@ -204,7 +205,7 @@ class BulkData(object):
self.getnode.cache_remove(self, oldunicodepath)

# Move the table to a temporary location
tmpdir = tempfile.mkdtemp(prefix = b"rename-", dir = self.root)
tmpdir = tempfile.mkdtemp(prefix=b"rename-", dir=self.root)
tmppath = os.path.join(tmpdir, b"table")
os.rename(oldospath, tmppath)

@@ -242,7 +243,7 @@ class BulkData(object):
# Remove the contents of the target directory
if not Table.exists(ospath):
raise ValueError("nothing at that path")
for (root, dirs, files) in os.walk(ospath, topdown = False):
for (root, dirs, files) in os.walk(ospath, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
@@ -252,8 +253,8 @@ class BulkData(object):
self._remove_leaves(unicodepath)

# Cache open tables
@nilmdb.utils.lru_cache(size = table_cache_size,
onremove = lambda x: x.close())
@nilmdb.utils.lru_cache(size=table_cache_size,
onremove=lambda x: x.close())
def getnode(self, unicodepath):
"""Return a Table object corresponding to the given database
path, which must exist."""
@@ -262,7 +263,8 @@ class BulkData(object):
ospath = os.path.join(self.root, *elements)
return Table(ospath, self.initial_nrows)

@nilmdb.utils.must_close(wrap_verify = False)

@nilmdb.utils.must_close(wrap_verify=False)
class Table(object):
"""Tools to help access a single table (data at a specific OS path)."""
# See design.md for design details
@@ -289,15 +291,17 @@ class Table(object):
rows_per_file = max(file_size // rkt.binary_size, 1)
rkt.close()

fmt = { "rows_per_file": rows_per_file,
"files_per_dir": files_per_dir,
"layout": layout,
"version": 3 }
fmt = {
"rows_per_file": rows_per_file,
"files_per_dir": files_per_dir,
"layout": layout,
"version": 3
}
with open(os.path.join(root, b"_format"), "wb") as f:
pickle.dump(fmt, f, 2)

# Normal methods
def __init__(self, root, initial_nrows = 0):
def __init__(self, root, initial_nrows=0):
"""'root' is the full OS path to the directory of this table"""
self.root = root
self.initial_nrows = initial_nrows
@@ -343,7 +347,7 @@ class Table(object):
# empty if something was deleted but the directory was unexpectedly
# not deleted.
subdirs = sorted(filter(regex.search, os.listdir(self.root)),
key = lambda x: int(x, 16), reverse = True)
key=lambda x: int(x, 16), reverse=True)

for subdir in subdirs:
# Now find the last file in that dir
@@ -354,7 +358,7 @@ class Table(object):
continue

# Find the numerical max
filename = max(files, key = lambda x: int(x, 16))
filename = max(files, key=lambda x: int(x, 16))
offset = os.path.getsize(os.path.join(self.root, subdir, filename))

# Convert to row number
@@ -396,7 +400,7 @@ class Table(object):
row = (filenum * self.rows_per_file) + (offset // self.row_size)
return row

def _remove_or_truncate_file(self, subdir, filename, offset = 0):
def _remove_or_truncate_file(self, subdir, filename, offset=0):
"""Remove the given file, and remove the subdirectory too
if it's empty. If offset is nonzero, truncate the file
to that size instead."""
@@ -416,8 +420,8 @@ class Table(object):
pass

# Cache open files
@nilmdb.utils.lru_cache(size = fd_cache_size,
onremove = lambda f: f.close())
@nilmdb.utils.lru_cache(size=fd_cache_size,
onremove=lambda f: f.close())
def file_open(self, subdir, filename):
"""Open and map a given 'subdir/filename' (relative to self.root).
Will be automatically closed when evicted from the cache."""
@@ -430,7 +434,7 @@ class Table(object):
return rocket.Rocket(self.layout,
os.path.join(self.root, subdir, filename))

def append_data(self, data, start, end, binary = False):
def append_data(self, data, start, end, binary=False):
"""Parse the formatted string in 'data', according to the
current layout, and append it to the table. If any timestamps
are non-monotonic, or don't fall between 'start' and 'end',
@@ -454,7 +458,7 @@ class Table(object):
while data_offset < len(data):
# See how many rows we can fit into the current file,
# and open it
(subdir, fname, offset, count) = self._offset_from_row(tot_rows)
(subdir, fname, offs, count) = self._offset_from_row(tot_rows)
f = self.file_open(subdir, fname)

# Ask the rocket object to parse and append up to "count"
@@ -503,8 +507,8 @@ class Table(object):
# deleting files that we may have appended data to.
cleanpos = self.nrows
while cleanpos <= tot_rows:
(subdir, fname, offset, count) = self._offset_from_row(cleanpos)
self._remove_or_truncate_file(subdir, fname, offset)
(subdir, fname, offs, count) = self._offset_from_row(cleanpos)
self._remove_or_truncate_file(subdir, fname, offs)
cleanpos += count
# Re-raise original exception
raise
@@ -512,14 +516,11 @@ class Table(object):
# Success, so update self.nrows accordingly
self.nrows = tot_rows

def get_data(self, start, stop, binary = False):
def get_data(self, start, stop, binary=False):
"""Extract data corresponding to Python range [n:m],
and returns a formatted string"""
if (start is None or
stop is None or
start > stop or
start < 0 or
stop > self.nrows):
if (start is None or stop is None or
start > stop or start < 0 or stop > self.nrows):
raise IndexError("Index out of range")

ret = []
@@ -597,7 +598,7 @@ class Table(object):
# remainder will be filled on a subsequent append(), and things
# are generally easier if we don't have to special-case that.
if (len(merged) == 1 and
merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
# Delete files
if cachefile_present:
os.remove(cachefile)


+ 4
- 1
nilmdb/server/errors.py View File

@@ -1,12 +1,15 @@
"""Exceptions"""


class NilmDBError(Exception):
"""Base exception for NilmDB errors"""
def __init__(self, msg = "Unspecified error"):
def __init__(self, msg="Unspecified error"):
super().__init__(msg)


class StreamError(NilmDBError):
pass


class OverlapError(NilmDBError):
pass

+ 29
- 27
nilmdb/server/nilmdb.py View File

@@ -11,7 +11,7 @@ Manages both the SQL database and the table storage backend.
# nilmdb.py, but will pull the parent nilmdb module instead.

import nilmdb.utils
from nilmdb.utils.printf import *
from nilmdb.utils.printf import printf
from nilmdb.utils.time import timestamp_to_bytes

from nilmdb.utils.interval import IntervalError
@@ -37,7 +37,7 @@ import errno
# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL'
# give an equivalent speedup more safely. That is what is used here.
_sql_schema_updates = {
0: { "next": 1, "sql": """
0: {"next": 1, "sql": """
-- All streams
CREATE TABLE streams(
id INTEGER PRIMARY KEY, -- stream ID
@@ -61,23 +61,24 @@ _sql_schema_updates = {
end_pos INTEGER NOT NULL
);
CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time);
""" },
"""},

1: { "next": 3, "sql": """
1: {"next": 3, "sql": """
-- Generic dictionary-type metadata that can be associated with a stream
CREATE TABLE metadata(
stream_id INTEGER NOT NULL,
stream_id INTEGER NOT NULL,
key TEXT NOT NULL,
value TEXT
);
""" },
"""},

2: { "error": "old format with floating-point timestamps requires "
"nilmdb 1.3.1 or older" },
2: {"error": "old format with floating-point timestamps requires "
"nilmdb 1.3.1 or older"},

3: { "next": None },
3: {"next": None},
}


@nilmdb.utils.must_close()
class NilmDB(object):
verbose = 0
@@ -119,7 +120,7 @@ class NilmDB(object):

# SQLite database too
sqlfilename = os.path.join(self.basepath, "data.sql")
self.con = sqlite3.connect(sqlfilename, check_same_thread = True)
self.con = sqlite3.connect(sqlfilename, check_same_thread=True)
try:
self._sql_schema_update()
except Exception:
@@ -183,7 +184,7 @@ class NilmDB(object):
raise NilmDBError("start must precede end")
return (start, end)

@nilmdb.utils.lru_cache(size = 64)
@nilmdb.utils.lru_cache(size=64)
def _get_intervals(self, stream_id):
"""
Return a mutable IntervalSet corresponding to the given stream ID.
@@ -231,11 +232,11 @@ class NilmDB(object):
# time range [adjacent.start -> interval.end)
# and database rows [ adjacent.start_pos -> end_pos ].
# Only do this if the resulting interval isn't too large.
max_merged_rows = 8000 * 60 * 60 * 1.05 # 1.05 hours at 8 KHz
max_merged_rows = 8000 * 60 * 60 * 1.05 # 1.05 hours at 8 KHz
adjacent = iset.find_end(interval.start)
if (adjacent is not None and
start_pos == adjacent.db_endpos and
(end_pos - adjacent.db_startpos) < max_merged_rows):
start_pos == adjacent.db_endpos and
(end_pos - adjacent.db_startpos) < max_merged_rows):
# First delete the old one, both from our iset and the
# database
iset -= adjacent
@@ -281,7 +282,8 @@ class NilmDB(object):
# the removed piece was in the middle.
def add(iset, start, end, start_pos, end_pos):
iset += DBInterval(start, end, start, end, start_pos, end_pos)
self._sql_interval_insert(stream_id, start, end, start_pos, end_pos)
self._sql_interval_insert(stream_id, start, end,
start_pos, end_pos)

if original.start != remove.start:
# Interval before the removed region
@@ -298,7 +300,7 @@ class NilmDB(object):

return

def stream_list(self, path = None, layout = None, extended = False):
def stream_list(self, path=None, layout=None, extended=False):
"""Return list of lists of all streams in the database.

If path is specified, include only streams with a path that
@@ -307,10 +309,10 @@ class NilmDB(object):
If layout is specified, include only streams with a layout
that matches the given string.

If extended = False, returns a list of lists containing
If extended=False, returns a list of lists containing
the path and layout: [ path, layout ]

If extended = True, returns a list of lists containing
If extended=True, returns a list of lists containing
more information:
path
layout
@@ -337,9 +339,9 @@ class NilmDB(object):
params += (path,)
query += " GROUP BY streams.id ORDER BY streams.path"
result = self.con.execute(query, params).fetchall()
return [ list(x) for x in result ]
return [list(x) for x in result]

def stream_intervals(self, path, start = None, end = None, diffpath = None):
def stream_intervals(self, path, start=None, end=None, diffpath=None):
"""
List all intervals in 'path' between 'start' and 'end'. If
'diffpath' is not none, list instead the set-difference
@@ -411,8 +413,8 @@ class NilmDB(object):

def stream_set_metadata(self, path, data):
"""Set stream metadata from a dictionary, e.g.
{ description = 'Downstairs lighting',
v_scaling = 123.45 }
{ description: 'Downstairs lighting',
v_scaling: 123.45 }
This replaces all existing metadata.
"""
stream_id = self._stream_id(path)
@@ -474,7 +476,7 @@ class NilmDB(object):
con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
con.execute("DELETE FROM streams WHERE id=?", (stream_id,))

def stream_insert(self, path, start, end, data, binary = False):
def stream_insert(self, path, start, end, data, binary=False):
"""Insert new data into the database.
path: Path at which to add the data
start: Starting timestamp
@@ -551,8 +553,8 @@ class NilmDB(object):
dbinterval.db_startpos,
dbinterval.db_endpos)

def stream_extract(self, path, start = None, end = None,
count = False, markup = False, binary = False):
def stream_extract(self, path, start=None, end=None,
count=False, markup=False, binary=False):
"""
Returns (data, restart) tuple.

@@ -632,7 +634,7 @@ class NilmDB(object):
full_result = b"".join(result)
return (full_result, restart)

def stream_remove(self, path, start = None, end = None):
def stream_remove(self, path, start=None, end=None):
"""
Remove data from the specified time interval within a stream.

@@ -659,7 +661,7 @@ class NilmDB(object):

# Can't remove intervals from within the iterator, so we need to
# remember what's currently in the intersection now.
all_candidates = list(intervals.intersection(to_remove, orig = True))
all_candidates = list(intervals.intersection(to_remove, orig=True))

remove_start = None
remove_end = None


+ 63
- 51
nilmdb/server/server.py View File

@@ -4,16 +4,14 @@
# nilmdb.py, but will pull the nilmdb module instead.

import nilmdb.server
from nilmdb.utils.printf import *
from nilmdb.utils.printf import sprintf
from nilmdb.server.errors import NilmDBError
from nilmdb.utils.time import string_to_timestamp

import cherrypy
import sys
import os
import socket
import json
import decorator
import psutil
import traceback

@@ -32,10 +30,12 @@ from nilmdb.server.serverutil import (
# Add CORS_allow tool
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)


class NilmApp(object):
def __init__(self, db):
self.db = db


# CherryPy apps
class Root(NilmApp):
"""Root application for NILM database"""
@@ -71,11 +71,14 @@ class Root(NilmApp):
path = self.db.get_basepath()
usage = psutil.disk_usage(path)
dbsize = nilmdb.utils.du(path)
return { "path": path,
"size": dbsize,
"other": max(usage.used - dbsize, 0),
"reserved": max(usage.total - usage.used - usage.free, 0),
"free": usage.free }
return {
"path": path,
"size": dbsize,
"other": max(usage.used - dbsize, 0),
"reserved": max(usage.total - usage.used - usage.free, 0),
"free": usage.free
}


class Stream(NilmApp):
"""Stream-specific operations"""
@@ -88,7 +91,8 @@ class Stream(NilmApp):
start = string_to_timestamp(start_param)
except Exception:
raise cherrypy.HTTPError("400 Bad Request", sprintf(
"invalid start (%s): must be a numeric timestamp", start_param))