Compare commits
No commits in common. "master" and "python2" have entirely different histories.
|
|
@ -1,11 +1,10 @@
|
|||
# -*- conf -*-
|
||||
|
||||
[run]
|
||||
branch = True
|
||||
# branch = True
|
||||
|
||||
[report]
|
||||
exclude_lines =
|
||||
pragma: no cover
|
||||
if 0:
|
||||
omit = nilmdb/scripts,nilmdb/_version.py,nilmdb/fsck
|
||||
show_missing = True
|
||||
omit = nilmdb/utils/datetime_tz*,nilmdb/scripts,nilmdb/_version.py,nilmdb/fsck
|
||||
|
|
|
|||
7
.gitignore
vendored
7
.gitignore
vendored
|
|
@ -4,7 +4,6 @@ tests/*testdb/
|
|||
db/
|
||||
|
||||
# Compiled / cythonized files
|
||||
README.html
|
||||
docs/*.html
|
||||
build/
|
||||
*.pyc
|
||||
|
|
@ -16,8 +15,10 @@ nilmdb/server/rbtree.c
|
|||
# Setup junk
|
||||
dist/
|
||||
nilmdb.egg-info/
|
||||
venv/
|
||||
.eggs/
|
||||
|
||||
# This gets generated as needed by setup.py
|
||||
MANIFEST.in
|
||||
MANIFEST
|
||||
|
||||
# Misc
|
||||
timeit*out
|
||||
|
|
|
|||
250
.pylintrc
Normal file
250
.pylintrc
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
# -*- conf -*-
|
||||
[MASTER]
|
||||
|
||||
# Specify a configuration file.
|
||||
#rcfile=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Profiled execution.
|
||||
profile=no
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=datetime_tz
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=no
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once).
|
||||
disable=C0111,R0903,R0201,R0914,R0912,W0142,W0703,W0702
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html
|
||||
output-format=parseable
|
||||
|
||||
# Include message's id in output
|
||||
include-ids=yes
|
||||
|
||||
# Put messages in a separate file for each module / package specified on the
|
||||
# command line instead of printing them on stdout. Reports (if any) will be
|
||||
# written in a file name "pylint_global.[txt|html]".
|
||||
files-output=no
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=yes
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Add a comment according to your evaluation note. This is used by the global
|
||||
# evaluation report (RP0004).
|
||||
comment=no
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||
ignore-mixin-members=yes
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set).
|
||||
ignored-classes=SQLObject
|
||||
|
||||
# When zope mode is activated, add a predefined set of Zope acquired attributes
|
||||
# to generated-members.
|
||||
zope=no
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E0201 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=REQUEST,acl_users,aq_parent
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=80
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=1000
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,XXX,TODO
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the beginning of the name of dummy variables
|
||||
# (i.e. not used).
|
||||
dummy-variables-rgx=_|dummy
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# Required attributes for module, separated by a comma
|
||||
required-attributes=
|
||||
|
||||
# List of builtins function names that should not be used, separated by a comma
|
||||
bad-functions=apply,input
|
||||
|
||||
# Regular expression which should only match correct module names
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Regular expression which should only match correct module level names
|
||||
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|version)$
|
||||
|
||||
# Regular expression which should only match correct class names
|
||||
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Regular expression which should only match correct function names
|
||||
function-rgx=[a-z_][a-z0-9_]{0,30}$
|
||||
|
||||
# Regular expression which should only match correct method names
|
||||
method-rgx=[a-z_][a-z0-9_]{0,30}$
|
||||
|
||||
# Regular expression which should only match correct instance attribute names
|
||||
attr-rgx=[a-z_][a-z0-9_]{0,30}$
|
||||
|
||||
# Regular expression which should only match correct argument names
|
||||
argument-rgx=[a-z_][a-z0-9_]{0,30}$
|
||||
|
||||
# Regular expression which should only match correct variable names
|
||||
variable-rgx=[a-z_][a-z0-9_]{0,30}$
|
||||
|
||||
# Regular expression which should only match correct list comprehension /
|
||||
# generator expression variable names
|
||||
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
good-names=i,j,k,ex,Run,_
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=foo,bar,baz,toto,tutu,tata
|
||||
|
||||
# Regular expression which should only match functions or classes name which do
|
||||
# not require a docstring
|
||||
no-docstring-rgx=__.*__
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of interface methods to ignore, separated by a comma. This is used for
|
||||
# instance to not check methods defines in Zope's Interface base class.
|
||||
ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,__new__,setUp
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
ignored-argument-names=_.*
|
||||
|
||||
# Maximum number of locals for function / method body
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of return / yield for function / method body
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of branch for function / method body
|
||||
max-branchs=12
|
||||
|
||||
# Maximum number of statements in function / method body
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when being caught. Defaults to
|
||||
# "Exception"
|
||||
overgeneral-exceptions=Exception
|
||||
29
MANIFEST.in
29
MANIFEST.in
|
|
@ -1,29 +0,0 @@
|
|||
# Root
|
||||
include README.txt
|
||||
include setup.cfg
|
||||
include setup.py
|
||||
include versioneer.py
|
||||
include Makefile
|
||||
include .coveragerc
|
||||
include .pylintrc
|
||||
include requirements.txt
|
||||
|
||||
# Cython files -- include .pyx source, but not the generated .c files
|
||||
# (Downstream systems must have cython installed in order to build)
|
||||
recursive-include nilmdb/server *.pyx *.pyxdep *.pxd
|
||||
exclude nilmdb/server/interval.c
|
||||
exclude nilmdb/server/rbtree.c
|
||||
|
||||
# Version
|
||||
include nilmdb/_version.py
|
||||
|
||||
# Tests
|
||||
recursive-include tests *.py
|
||||
recursive-include tests/data *
|
||||
include tests/test.order
|
||||
|
||||
# Docs
|
||||
recursive-include docs Makefile *.md
|
||||
|
||||
# Extras
|
||||
recursive-include extras *
|
||||
30
Makefile
30
Makefile
|
|
@ -2,49 +2,45 @@
|
|||
all: test
|
||||
|
||||
version:
|
||||
python3 setup.py version
|
||||
python setup.py version
|
||||
|
||||
build:
|
||||
python3 setup.py build_ext --inplace
|
||||
python setup.py build_ext --inplace
|
||||
|
||||
dist: sdist
|
||||
sdist:
|
||||
python3 setup.py sdist
|
||||
python setup.py sdist
|
||||
|
||||
install:
|
||||
python3 setup.py install
|
||||
python setup.py install
|
||||
|
||||
develop:
|
||||
python3 setup.py develop
|
||||
python setup.py develop
|
||||
|
||||
docs:
|
||||
make -C docs
|
||||
|
||||
ctrl: flake
|
||||
flake:
|
||||
flake8 nilmdb
|
||||
lint:
|
||||
pylint3 --rcfile=setup.cfg nilmdb
|
||||
pylint --rcfile=.pylintrc nilmdb
|
||||
|
||||
test:
|
||||
ifneq ($(INSIDE_EMACS),)
|
||||
ifeq ($(INSIDE_EMACS), t)
|
||||
# Use the slightly more flexible script
|
||||
python3 setup.py build_ext --inplace
|
||||
python3 tests/runtests.py
|
||||
python setup.py build_ext --inplace
|
||||
python tests/runtests.py
|
||||
else
|
||||
# Let setup.py check dependencies, build stuff, and run the test
|
||||
python3 setup.py nosetests
|
||||
python setup.py nosetests
|
||||
endif
|
||||
|
||||
clean::
|
||||
find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
|
||||
find . -name '*pyc' | xargs rm -f
|
||||
rm -f .coverage
|
||||
rm -rf tests/*testdb*
|
||||
rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so
|
||||
rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so MANIFEST.in
|
||||
make -C docs clean
|
||||
|
||||
gitclean::
|
||||
git clean -dXf
|
||||
|
||||
.PHONY: all version build dist sdist install docs test
|
||||
.PHONY: ctrl lint flake clean gitclean
|
||||
.PHONY: all version build dist sdist install docs lint test clean gitclean
|
||||
|
|
|
|||
40
README.md
40
README.md
|
|
@ -1,40 +0,0 @@
|
|||
# nilmdb: Non-Intrusive Load Monitor Database
|
||||
by Jim Paris <jim@jtan.com>
|
||||
|
||||
NilmDB requires Python 3.8 or newer.
|
||||
|
||||
## Prerequisites:
|
||||
|
||||
# Runtime and build environments
|
||||
sudo apt install python3 python3-dev python3-venv python3-pip
|
||||
|
||||
# Create a new Python virtual environment to isolate deps.
|
||||
python3 -m venv ../venv
|
||||
source ../venv/bin/activate # run "deactivate" to leave
|
||||
|
||||
# Install all Python dependencies
|
||||
pip3 install -r requirements.txt
|
||||
|
||||
## Test:
|
||||
|
||||
python3 setup.py nosetests
|
||||
|
||||
## Install:
|
||||
|
||||
Install it into the virtual environment
|
||||
|
||||
python3 setup.py install
|
||||
|
||||
If you want to instead install it system-wide, you will also need to
|
||||
install the requirements system-wide:
|
||||
|
||||
sudo pip3 install -r requirements.txt
|
||||
sudo python3 setup.py install
|
||||
|
||||
## Usage:
|
||||
|
||||
nilmdb-server --help
|
||||
nilmdb-fsck --help
|
||||
nilmtool --help
|
||||
|
||||
See docs/wsgi.md for info on setting up a WSGI application in Apache.
|
||||
33
README.txt
Normal file
33
README.txt
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
nilmdb: Non-Intrusive Load Monitor Database
|
||||
by Jim Paris <jim@jtan.com>
|
||||
|
||||
Prerequisites:
|
||||
|
||||
# Runtime and build environments
|
||||
sudo apt-get install python2.7 python2.7-dev python-setuptools cython
|
||||
|
||||
# Base NilmDB dependencies
|
||||
sudo apt-get install python-cherrypy3 python-decorator python-simplejson
|
||||
sudo apt-get install python-requests python-dateutil python-tz
|
||||
sudo apt-get install python-progressbar python-psutil
|
||||
|
||||
# Other dependencies (required by some modules)
|
||||
sudo apt-get install python-numpy
|
||||
|
||||
# Tools for running tests
|
||||
sudo apt-get install python-nose python-coverage
|
||||
|
||||
Test:
|
||||
python setup.py nosetests
|
||||
|
||||
Install:
|
||||
|
||||
python setup.py install
|
||||
|
||||
Usage:
|
||||
|
||||
nilmdb-server --help
|
||||
nilmdb-fsck --help
|
||||
nilmtool --help
|
||||
|
||||
See docs/wsgi.md for info on setting up a WSGI application in Apache.
|
||||
|
|
@ -430,7 +430,7 @@ mod_wsgi requires "WSGIChunkedRequest On" to handle
|
|||
"Transfer-encoding: Chunked" requests. However, `/stream/insert`
|
||||
doesn't handle this correctly right now, because:
|
||||
|
||||
- The `cherrypy.request.body.read()` call needs to be fixed for chunked requests
|
||||
- The `cherrpy.request.body.read()` call needs to be fixed for chunked requests
|
||||
|
||||
- We don't want to just buffer endlessly in the server, and it will
|
||||
require some thought on how to handle data in chunks (what to do about
|
||||
|
|
@ -438,32 +438,3 @@ doesn't handle this correctly right now, because:
|
|||
|
||||
It is probably better to just keep the endpoint management on the client
|
||||
side, so leave "WSGIChunkedRequest off" for now.
|
||||
|
||||
|
||||
Unicode & character encoding
|
||||
----------------------------
|
||||
|
||||
Stream data is passed back and forth as raw `bytes` objects in most
|
||||
places, including the `nilmdb.client` and command-line interfaces.
|
||||
This is done partially for performance reasons, and partially to
|
||||
support the binary insert/extract options, where character-set encoding
|
||||
would not apply.
|
||||
|
||||
For the HTTP server, the raw bytes transferred over HTTP are interpreted
|
||||
as follows:
|
||||
- For `/stream/insert`, the client-provided `Content-Type` is ignored,
|
||||
and the data is read as if it were `application/octet-stream`.
|
||||
- For `/stream/extract`, the returned data is `application/octet-stream`.
|
||||
- All other endpoints communicate via JSON, which is specified to always
|
||||
be encoded as UTF-8. This includes:
|
||||
- `/version`
|
||||
- `/dbinfo`
|
||||
- `/stream/list`
|
||||
- `/stream/create`
|
||||
- `/stream/destroy`
|
||||
- `/stream/rename`
|
||||
- `/stream/get_metadata`
|
||||
- `/stream/set_metadata`
|
||||
- `/stream/update_metadata`
|
||||
- `/stream/remove`
|
||||
- `/stream/intervals`
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
#!/usr/bin/env python3
|
||||
#!/usr/bin/python
|
||||
|
||||
import os
|
||||
import sys
|
||||
import pickle
|
||||
import cPickle as pickle
|
||||
import argparse
|
||||
import fcntl
|
||||
import re
|
||||
|
|
@ -44,7 +44,7 @@ with open(lock, "w") as f:
|
|||
maxsize = fix[fixpath]
|
||||
if size > maxsize:
|
||||
diff = size - maxsize
|
||||
print(diff, "too big:", fn)
|
||||
print diff, "too big:", fn
|
||||
if args.yes:
|
||||
with open(fn, "a+") as dbfile:
|
||||
dbfile.truncate(maxsize)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
"""Main NilmDB import"""
|
||||
|
||||
from ._version import get_versions
|
||||
# These aren't imported automatically, because loading the server
|
||||
# stuff isn't always necessary.
|
||||
#from nilmdb.server import NilmDB, Server
|
||||
#from nilmdb.client import Client
|
||||
|
||||
from nilmdb._version import get_versions
|
||||
__version__ = get_versions()['version']
|
||||
del get_versions
|
||||
|
|
|
|||
|
|
@ -1,520 +1,197 @@
|
|||
|
||||
IN_LONG_VERSION_PY = True
|
||||
# This file helps to compute a version number in source trees obtained from
|
||||
# git-archive tarball (such as those provided by githubs download-from-tag
|
||||
# feature). Distribution tarballs (built by setup.py sdist) and build
|
||||
# feature). Distribution tarballs (build by setup.py sdist) and build
|
||||
# directories (produced by setup.py build) will contain a much shorter file
|
||||
# that just contains the computed version number.
|
||||
|
||||
# This file is released into the public domain. Generated by
|
||||
# versioneer-0.18 (https://github.com/warner/python-versioneer)
|
||||
# versioneer-0.7+ (https://github.com/warner/python-versioneer)
|
||||
|
||||
# these strings will be replaced by git during git-archive
|
||||
git_refnames = "$Format:%d$"
|
||||
git_full = "$Format:%H$"
|
||||
|
||||
"""Git implementation of _version.py."""
|
||||
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def get_keywords():
|
||||
"""Get the keywords needed to look up the version information."""
|
||||
# these strings will be replaced by git during git-archive.
|
||||
# setup.py/versioneer.py will grep for the variable names, so they must
|
||||
# each be defined on a line of their own. _version.py will just call
|
||||
# get_keywords().
|
||||
git_refnames = "$Format:%d$"
|
||||
git_full = "$Format:%H$"
|
||||
git_date = "$Format:%ci$"
|
||||
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
|
||||
return keywords
|
||||
|
||||
|
||||
class VersioneerConfig:
|
||||
"""Container for Versioneer configuration parameters."""
|
||||
|
||||
|
||||
def get_config():
|
||||
"""Create, populate and return the VersioneerConfig() object."""
|
||||
# these strings are filled in when 'setup.py versioneer' creates
|
||||
# _version.py
|
||||
cfg = VersioneerConfig()
|
||||
cfg.VCS = "git"
|
||||
cfg.style = "pep440"
|
||||
cfg.tag_prefix = "nilmdb-"
|
||||
cfg.parentdir_prefix = "nilmdb-"
|
||||
cfg.versionfile_source = "nilmdb/_version.py"
|
||||
cfg.verbose = False
|
||||
return cfg
|
||||
|
||||
|
||||
class NotThisMethod(Exception):
|
||||
"""Exception raised if a method is not valid for the current scenario."""
|
||||
|
||||
|
||||
LONG_VERSION_PY = {}
|
||||
HANDLERS = {}
|
||||
|
||||
|
||||
def register_vcs_handler(vcs, method): # decorator
|
||||
"""Decorator to mark a method as the handler for a particular VCS."""
|
||||
def decorate(f):
|
||||
"""Store f in HANDLERS[vcs][method]."""
|
||||
if vcs not in HANDLERS:
|
||||
HANDLERS[vcs] = {}
|
||||
HANDLERS[vcs][method] = f
|
||||
return f
|
||||
return decorate
|
||||
|
||||
|
||||
def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
|
||||
env=None):
|
||||
"""Call the given command(s)."""
|
||||
assert isinstance(commands, list)
|
||||
p = None
|
||||
for c in commands:
|
||||
def run_command(args, cwd=None, verbose=False):
|
||||
try:
|
||||
dispcmd = str([c] + args)
|
||||
# remember shell=False, so use git.cmd on windows, not just git
|
||||
p = subprocess.Popen([c] + args, cwd=cwd, env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=(subprocess.PIPE if hide_stderr
|
||||
else None))
|
||||
break
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
|
||||
except EnvironmentError:
|
||||
e = sys.exc_info()[1]
|
||||
if e.errno == errno.ENOENT:
|
||||
continue
|
||||
if verbose:
|
||||
print("unable to run %s" % dispcmd)
|
||||
print("unable to run %s" % args[0])
|
||||
print(e)
|
||||
return None, None
|
||||
else:
|
||||
if verbose:
|
||||
print("unable to find command, tried %s" % (commands,))
|
||||
return None, None
|
||||
return None
|
||||
stdout = p.communicate()[0].strip()
|
||||
if sys.version_info[0] >= 3:
|
||||
if sys.version >= '3':
|
||||
stdout = stdout.decode()
|
||||
if p.returncode != 0:
|
||||
if verbose:
|
||||
print("unable to run %s (error)" % dispcmd)
|
||||
print("stdout was %s" % stdout)
|
||||
return None, p.returncode
|
||||
return stdout, p.returncode
|
||||
print("unable to run %s (error)" % args[0])
|
||||
return None
|
||||
return stdout
|
||||
|
||||
|
||||
def versions_from_parentdir(parentdir_prefix, root, verbose):
|
||||
"""Try to determine the version from the parent directory name.
|
||||
import sys
|
||||
import re
|
||||
import os.path
|
||||
|
||||
Source tarballs conventionally unpack into a directory that includes both
|
||||
the project name and a version string. We will also support searching up
|
||||
two directory levels for an appropriately named parent directory
|
||||
"""
|
||||
rootdirs = []
|
||||
|
||||
for i in range(3):
|
||||
dirname = os.path.basename(root)
|
||||
if dirname.startswith(parentdir_prefix):
|
||||
return {"version": dirname[len(parentdir_prefix):],
|
||||
"full-revisionid": None,
|
||||
"dirty": False, "error": None, "date": None}
|
||||
else:
|
||||
rootdirs.append(root)
|
||||
root = os.path.dirname(root) # up a level
|
||||
|
||||
if verbose:
|
||||
print("Tried directories %s but none started with prefix %s" %
|
||||
(str(rootdirs), parentdir_prefix))
|
||||
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
|
||||
|
||||
|
||||
@register_vcs_handler("git", "get_keywords")
|
||||
def git_get_keywords(versionfile_abs):
|
||||
"""Extract version information from the given file."""
|
||||
def get_expanded_variables(versionfile_source):
|
||||
# the code embedded in _version.py can just fetch the value of these
|
||||
# keywords. When used from setup.py, we don't want to import _version.py,
|
||||
# so we do it with a regexp instead. This function is not used from
|
||||
# _version.py.
|
||||
keywords = {}
|
||||
# variables. When used from setup.py, we don't want to import
|
||||
# _version.py, so we do it with a regexp instead. This function is not
|
||||
# used from _version.py.
|
||||
variables = {}
|
||||
try:
|
||||
f = open(versionfile_abs, "r")
|
||||
for line in f.readlines():
|
||||
for line in open(versionfile_source,"r").readlines():
|
||||
if line.strip().startswith("git_refnames ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["refnames"] = mo.group(1)
|
||||
variables["refnames"] = mo.group(1)
|
||||
if line.strip().startswith("git_full ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["full"] = mo.group(1)
|
||||
if line.strip().startswith("git_date ="):
|
||||
mo = re.search(r'=\s*"(.*)"', line)
|
||||
if mo:
|
||||
keywords["date"] = mo.group(1)
|
||||
f.close()
|
||||
variables["full"] = mo.group(1)
|
||||
except EnvironmentError:
|
||||
pass
|
||||
return keywords
|
||||
return variables
|
||||
|
||||
|
||||
@register_vcs_handler("git", "keywords")
|
||||
def git_versions_from_keywords(keywords, tag_prefix, verbose):
|
||||
"""Get version information from git keywords."""
|
||||
if not keywords:
|
||||
raise NotThisMethod("no keywords at all, weird")
|
||||
date = keywords.get("date")
|
||||
if date is not None:
|
||||
# git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
|
||||
# datestamp. However we prefer "%ci" (which expands to an "ISO-8601
|
||||
# -like" string, which we must then edit to make compliant), because
|
||||
# it's been around since git-1.5.3, and it's too difficult to
|
||||
# discover which version we're using, or to work around using an
|
||||
# older one.
|
||||
date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
|
||||
refnames = keywords["refnames"].strip()
|
||||
def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
|
||||
refnames = variables["refnames"].strip()
|
||||
if refnames.startswith("$Format"):
|
||||
if verbose:
|
||||
print("keywords are unexpanded, not using")
|
||||
raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
|
||||
print("variables are unexpanded, not using")
|
||||
return {} # unexpanded, so not in an unpacked git-archive tarball
|
||||
refs = set([r.strip() for r in refnames.strip("()").split(",")])
|
||||
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
|
||||
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
|
||||
TAG = "tag: "
|
||||
tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
|
||||
if not tags:
|
||||
# Either we're using git < 1.8.3, or there really are no tags. We use
|
||||
# a heuristic: assume all version tags have a digit. The old git %d
|
||||
# expansion behaves like git log --decorate=short and strips out the
|
||||
# refs/heads/ and refs/tags/ prefixes that would let us distinguish
|
||||
# between branches and tags. By ignoring refnames without digits, we
|
||||
# filter out many common branch names like "release" and
|
||||
# "stabilization", as well as "HEAD" and "master".
|
||||
tags = set([r for r in refs if re.search(r'\d', r)])
|
||||
for ref in list(refs):
|
||||
if not re.search(r'\d', ref):
|
||||
if verbose:
|
||||
print("discarding '%s', no digits" % ",".join(refs - tags))
|
||||
print("discarding '%s', no digits" % ref)
|
||||
refs.discard(ref)
|
||||
# Assume all version tags have a digit. git's %d expansion
|
||||
# behaves like git log --decorate=short and strips out the
|
||||
# refs/heads/ and refs/tags/ prefixes that would let us
|
||||
# distinguish between branches and tags. By ignoring refnames
|
||||
# without digits, we filter out many common branch names like
|
||||
# "release" and "stabilization", as well as "HEAD" and "master".
|
||||
if verbose:
|
||||
print("likely tags: %s" % ",".join(sorted(tags)))
|
||||
for ref in sorted(tags):
|
||||
print("remaining refs: %s" % ",".join(sorted(refs)))
|
||||
for ref in sorted(refs):
|
||||
# sorting will prefer e.g. "2.0" over "2.0rc1"
|
||||
if ref.startswith(tag_prefix):
|
||||
r = ref[len(tag_prefix):]
|
||||
if verbose:
|
||||
print("picking %s" % r)
|
||||
return {"version": r,
|
||||
"full-revisionid": keywords["full"].strip(),
|
||||
"dirty": False, "error": None,
|
||||
"date": date}
|
||||
# no suitable tags, so version is "0+unknown", but full hex is still there
|
||||
return { "version": r,
|
||||
"full": variables["full"].strip() }
|
||||
# no suitable tags, so we use the full revision id
|
||||
if verbose:
|
||||
print("no suitable tags, using unknown + full revision id")
|
||||
return {"version": "0+unknown",
|
||||
"full-revisionid": keywords["full"].strip(),
|
||||
"dirty": False, "error": "no suitable tags", "date": None}
|
||||
print("no suitable tags, using full revision id")
|
||||
return { "version": variables["full"].strip(),
|
||||
"full": variables["full"].strip() }
|
||||
|
||||
|
||||
@register_vcs_handler("git", "pieces_from_vcs")
|
||||
def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
||||
"""Get version from 'git describe' in the root of the source tree.
|
||||
|
||||
This only gets called if the git-archive 'subst' keywords were *not*
|
||||
expanded, and _version.py hasn't already been rewritten with a short
|
||||
version string, meaning we're inside a checked out source tree.
|
||||
"""
|
||||
GITS = ["git"]
|
||||
if sys.platform == "win32":
|
||||
GITS = ["git.cmd", "git.exe"]
|
||||
|
||||
out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
|
||||
hide_stderr=True)
|
||||
if rc != 0:
|
||||
if verbose:
|
||||
print("Directory %s not under git control" % root)
|
||||
raise NotThisMethod("'git rev-parse --git-dir' returned error")
|
||||
|
||||
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
|
||||
# if there isn't one, this yields HEX[-dirty] (no NUM)
|
||||
describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
|
||||
"--always", "--long",
|
||||
"--match", "%s*" % tag_prefix],
|
||||
cwd=root)
|
||||
# --long was added in git-1.5.5
|
||||
if describe_out is None:
|
||||
raise NotThisMethod("'git describe' failed")
|
||||
describe_out = describe_out.strip()
|
||||
full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
|
||||
if full_out is None:
|
||||
raise NotThisMethod("'git rev-parse' failed")
|
||||
full_out = full_out.strip()
|
||||
|
||||
pieces = {}
|
||||
pieces["long"] = full_out
|
||||
pieces["short"] = full_out[:7] # maybe improved later
|
||||
pieces["error"] = None
|
||||
|
||||
# parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
|
||||
# TAG might have hyphens.
|
||||
git_describe = describe_out
|
||||
|
||||
# look for -dirty suffix
|
||||
dirty = git_describe.endswith("-dirty")
|
||||
pieces["dirty"] = dirty
|
||||
if dirty:
|
||||
git_describe = git_describe[:git_describe.rindex("-dirty")]
|
||||
|
||||
# now we have TAG-NUM-gHEX or HEX
|
||||
|
||||
if "-" in git_describe:
|
||||
# TAG-NUM-gHEX
|
||||
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
|
||||
if not mo:
|
||||
# unparseable. Maybe git-describe is misbehaving?
|
||||
pieces["error"] = ("unable to parse git-describe output: '%s'"
|
||||
% describe_out)
|
||||
return pieces
|
||||
|
||||
# tag
|
||||
full_tag = mo.group(1)
|
||||
if not full_tag.startswith(tag_prefix):
|
||||
if verbose:
|
||||
fmt = "tag '%s' doesn't start with prefix '%s'"
|
||||
print(fmt % (full_tag, tag_prefix))
|
||||
pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
|
||||
% (full_tag, tag_prefix))
|
||||
return pieces
|
||||
pieces["closest-tag"] = full_tag[len(tag_prefix):]
|
||||
|
||||
# distance: number of commits since tag
|
||||
pieces["distance"] = int(mo.group(2))
|
||||
|
||||
# commit: short hex revision ID
|
||||
pieces["short"] = mo.group(3)
|
||||
|
||||
else:
|
||||
# HEX: no tags
|
||||
pieces["closest-tag"] = None
|
||||
count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
|
||||
cwd=root)
|
||||
pieces["distance"] = int(count_out) # total number of commits
|
||||
|
||||
# commit date: see ISO-8601 comment in git_versions_from_keywords()
|
||||
date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
|
||||
cwd=root)[0].strip()
|
||||
pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
|
||||
|
||||
return pieces
|
||||
|
||||
|
||||
def plus_or_dot(pieces):
|
||||
"""Return a + if we don't already have one, else return a ."""
|
||||
if "+" in pieces.get("closest-tag", ""):
|
||||
return "."
|
||||
return "+"
|
||||
|
||||
|
||||
def render_pep440(pieces):
|
||||
"""Build up version string, with post-release "local version identifier".
|
||||
|
||||
Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
|
||||
get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
|
||||
|
||||
Exceptions:
|
||||
1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
if pieces["distance"] or pieces["dirty"]:
|
||||
rendered += plus_or_dot(pieces)
|
||||
rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dirty"
|
||||
else:
|
||||
# exception #1
|
||||
rendered = "0+untagged.%d.g%s" % (pieces["distance"],
|
||||
pieces["short"])
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dirty"
|
||||
return rendered
|
||||
|
||||
|
||||
def render_pep440_pre(pieces):
|
||||
"""TAG[.post.devDISTANCE] -- No -dirty.
|
||||
|
||||
Exceptions:
|
||||
1: no tags. 0.post.devDISTANCE
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
if pieces["distance"]:
|
||||
rendered += ".post.dev%d" % pieces["distance"]
|
||||
else:
|
||||
# exception #1
|
||||
rendered = "0.post.dev%d" % pieces["distance"]
|
||||
return rendered
|
||||
|
||||
|
||||
def render_pep440_post(pieces):
|
||||
"""TAG[.postDISTANCE[.dev0]+gHEX] .
|
||||
|
||||
The ".dev0" means dirty. Note that .dev0 sorts backwards
|
||||
(a dirty tree will appear "older" than the corresponding clean one),
|
||||
but you shouldn't be releasing software with -dirty anyways.
|
||||
|
||||
Exceptions:
|
||||
1: no tags. 0.postDISTANCE[.dev0]
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
if pieces["distance"] or pieces["dirty"]:
|
||||
rendered += ".post%d" % pieces["distance"]
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dev0"
|
||||
rendered += plus_or_dot(pieces)
|
||||
rendered += "g%s" % pieces["short"]
|
||||
else:
|
||||
# exception #1
|
||||
rendered = "0.post%d" % pieces["distance"]
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dev0"
|
||||
rendered += "+g%s" % pieces["short"]
|
||||
return rendered
|
||||
|
||||
|
||||
def render_pep440_old(pieces):
|
||||
"""TAG[.postDISTANCE[.dev0]] .
|
||||
|
||||
The ".dev0" means dirty.
|
||||
|
||||
Eexceptions:
|
||||
1: no tags. 0.postDISTANCE[.dev0]
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
if pieces["distance"] or pieces["dirty"]:
|
||||
rendered += ".post%d" % pieces["distance"]
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dev0"
|
||||
else:
|
||||
# exception #1
|
||||
rendered = "0.post%d" % pieces["distance"]
|
||||
if pieces["dirty"]:
|
||||
rendered += ".dev0"
|
||||
return rendered
|
||||
|
||||
|
||||
def render_git_describe(pieces):
|
||||
"""TAG[-DISTANCE-gHEX][-dirty].
|
||||
|
||||
Like 'git describe --tags --dirty --always'.
|
||||
|
||||
Exceptions:
|
||||
1: no tags. HEX[-dirty] (note: no 'g' prefix)
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
if pieces["distance"]:
|
||||
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
|
||||
else:
|
||||
# exception #1
|
||||
rendered = pieces["short"]
|
||||
if pieces["dirty"]:
|
||||
rendered += "-dirty"
|
||||
return rendered
|
||||
|
||||
|
||||
def render_git_describe_long(pieces):
|
||||
"""TAG-DISTANCE-gHEX[-dirty].
|
||||
|
||||
Like 'git describe --tags --dirty --always -long'.
|
||||
The distance/hash is unconditional.
|
||||
|
||||
Exceptions:
|
||||
1: no tags. HEX[-dirty] (note: no 'g' prefix)
|
||||
"""
|
||||
if pieces["closest-tag"]:
|
||||
rendered = pieces["closest-tag"]
|
||||
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
|
||||
else:
|
||||
# exception #1
|
||||
rendered = pieces["short"]
|
||||
if pieces["dirty"]:
|
||||
rendered += "-dirty"
|
||||
return rendered
|
||||
|
||||
|
||||
def render(pieces, style):
|
||||
"""Render the given version pieces into the requested style."""
|
||||
if pieces["error"]:
|
||||
return {"version": "unknown",
|
||||
"full-revisionid": pieces.get("long"),
|
||||
"dirty": None,
|
||||
"error": pieces["error"],
|
||||
"date": None}
|
||||
|
||||
if not style or style == "default":
|
||||
style = "pep440" # the default
|
||||
|
||||
if style == "pep440":
|
||||
rendered = render_pep440(pieces)
|
||||
elif style == "pep440-pre":
|
||||
rendered = render_pep440_pre(pieces)
|
||||
elif style == "pep440-post":
|
||||
rendered = render_pep440_post(pieces)
|
||||
elif style == "pep440-old":
|
||||
rendered = render_pep440_old(pieces)
|
||||
elif style == "git-describe":
|
||||
rendered = render_git_describe(pieces)
|
||||
elif style == "git-describe-long":
|
||||
rendered = render_git_describe_long(pieces)
|
||||
else:
|
||||
raise ValueError("unknown style '%s'" % style)
|
||||
|
||||
return {"version": rendered, "full-revisionid": pieces["long"],
|
||||
"dirty": pieces["dirty"], "error": None,
|
||||
"date": pieces.get("date")}
|
||||
|
||||
|
||||
def get_versions():
|
||||
"""Get version information or return default if unable to do so."""
|
||||
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
|
||||
# __file__, we can work backwards from there to the root. Some
|
||||
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
|
||||
# case we can only use expanded keywords.
|
||||
|
||||
cfg = get_config()
|
||||
verbose = cfg.verbose
|
||||
def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
|
||||
# this runs 'git' from the root of the source tree. That either means
|
||||
# someone ran a setup.py command (and this code is in versioneer.py, so
|
||||
# IN_LONG_VERSION_PY=False, thus the containing directory is the root of
|
||||
# the source tree), or someone ran a project-specific entry point (and
|
||||
# this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
|
||||
# containing directory is somewhere deeper in the source tree). This only
|
||||
# gets called if the git-archive 'subst' variables were *not* expanded,
|
||||
# and _version.py hasn't already been rewritten with a short version
|
||||
# string, meaning we're inside a checked out source tree.
|
||||
|
||||
try:
|
||||
return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
|
||||
verbose)
|
||||
except NotThisMethod:
|
||||
pass
|
||||
|
||||
try:
|
||||
root = os.path.realpath(__file__)
|
||||
# versionfile_source is the relative path from the top of the source
|
||||
# tree (where the .git directory might live) to this file. Invert
|
||||
# this to find the root from __file__.
|
||||
for i in cfg.versionfile_source.split('/'):
|
||||
root = os.path.dirname(root)
|
||||
here = os.path.abspath(__file__)
|
||||
except NameError:
|
||||
return {"version": "0+unknown", "full-revisionid": None,
|
||||
"dirty": None,
|
||||
"error": "unable to find root of source tree",
|
||||
"date": None}
|
||||
# some py2exe/bbfreeze/non-CPython implementations don't do __file__
|
||||
return {} # not always correct
|
||||
|
||||
# versionfile_source is the relative path from the top of the source tree
|
||||
# (where the .git directory might live) to this file. Invert this to find
|
||||
# the root from __file__.
|
||||
root = here
|
||||
if IN_LONG_VERSION_PY:
|
||||
for i in range(len(versionfile_source.split("/"))):
|
||||
root = os.path.dirname(root)
|
||||
else:
|
||||
root = os.path.dirname(here)
|
||||
if not os.path.exists(os.path.join(root, ".git")):
|
||||
if verbose:
|
||||
print("no .git in %s" % root)
|
||||
return {}
|
||||
|
||||
GIT = "git"
|
||||
if sys.platform == "win32":
|
||||
GIT = "git.cmd"
|
||||
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
|
||||
cwd=root)
|
||||
if stdout is None:
|
||||
return {}
|
||||
if not stdout.startswith(tag_prefix):
|
||||
if verbose:
|
||||
print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
|
||||
return {}
|
||||
tag = stdout[len(tag_prefix):]
|
||||
stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
|
||||
if stdout is None:
|
||||
return {}
|
||||
full = stdout.strip()
|
||||
if tag.endswith("-dirty"):
|
||||
full += "-dirty"
|
||||
return {"version": tag, "full": full}
|
||||
|
||||
|
||||
def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
|
||||
if IN_LONG_VERSION_PY:
|
||||
# We're running from _version.py. If it's from a source tree
|
||||
# (execute-in-place), we can work upwards to find the root of the
|
||||
# tree, and then check the parent directory for a version string. If
|
||||
# it's in an installed application, there's no hope.
|
||||
try:
|
||||
pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
|
||||
return render(pieces, cfg.style)
|
||||
except NotThisMethod:
|
||||
pass
|
||||
here = os.path.abspath(__file__)
|
||||
except NameError:
|
||||
# py2exe/bbfreeze/non-CPython don't have __file__
|
||||
return {} # without __file__, we have no hope
|
||||
# versionfile_source is the relative path from the top of the source
|
||||
# tree to _version.py. Invert this to find the root from __file__.
|
||||
root = here
|
||||
for i in range(len(versionfile_source.split("/"))):
|
||||
root = os.path.dirname(root)
|
||||
else:
|
||||
# we're running from versioneer.py, which means we're running from
|
||||
# the setup.py in a source tree. sys.argv[0] is setup.py in the root.
|
||||
here = os.path.abspath(sys.argv[0])
|
||||
root = os.path.dirname(here)
|
||||
|
||||
try:
|
||||
if cfg.parentdir_prefix:
|
||||
return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
|
||||
except NotThisMethod:
|
||||
pass
|
||||
# Source tarballs conventionally unpack into a directory that includes
|
||||
# both the project name and a version string.
|
||||
dirname = os.path.basename(root)
|
||||
if not dirname.startswith(parentdir_prefix):
|
||||
if verbose:
|
||||
print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
|
||||
(root, dirname, parentdir_prefix))
|
||||
return None
|
||||
return {"version": dirname[len(parentdir_prefix):], "full": ""}
|
||||
|
||||
tag_prefix = "nilmdb-"
|
||||
parentdir_prefix = "nilmdb-"
|
||||
versionfile_source = "nilmdb/_version.py"
|
||||
|
||||
def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
|
||||
variables = { "refnames": git_refnames, "full": git_full }
|
||||
ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
|
||||
if not ver:
|
||||
ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
|
||||
if not ver:
|
||||
ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
|
||||
verbose)
|
||||
if not ver:
|
||||
ver = default
|
||||
return ver
|
||||
|
||||
return {"version": "0+unknown", "full-revisionid": None,
|
||||
"dirty": None,
|
||||
"error": "unable to compute version", "date": None}
|
||||
|
|
|
|||
|
|
@ -2,24 +2,24 @@
|
|||
|
||||
"""Class for performing HTTP client requests via libcurl"""
|
||||
|
||||
import json
|
||||
import contextlib
|
||||
|
||||
import nilmdb.utils
|
||||
import nilmdb.client.httpclient
|
||||
from nilmdb.client.errors import ClientError
|
||||
from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
|
||||
|
||||
import time
|
||||
import simplejson as json
|
||||
import contextlib
|
||||
|
||||
from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
|
||||
|
||||
def extract_timestamp(line):
|
||||
"""Extract just the timestamp from a line of data text"""
|
||||
return string_to_timestamp(line.split()[0])
|
||||
|
||||
|
||||
class Client():
|
||||
class Client(object):
|
||||
"""Main client interface to the Nilm database."""
|
||||
|
||||
def __init__(self, url, post_json=False):
|
||||
def __init__(self, url, post_json = False):
|
||||
"""Initialize client with given URL. If post_json is true,
|
||||
POST requests are sent with Content-Type 'application/json'
|
||||
instead of the default 'x-www-form-urlencoded'."""
|
||||
|
|
@ -38,7 +38,7 @@ class Client():
|
|||
if self.post_json:
|
||||
# If we're posting as JSON, we don't need to encode it further here
|
||||
return data
|
||||
return json.dumps(data, separators=(',', ':'))
|
||||
return json.dumps(data, separators=(',',':'))
|
||||
|
||||
def close(self):
|
||||
"""Close the connection; safe to call multiple times"""
|
||||
|
|
@ -57,7 +57,7 @@ class Client():
|
|||
as a dictionary."""
|
||||
return self.http.get("dbinfo")
|
||||
|
||||
def stream_list(self, path=None, layout=None, extended=False):
|
||||
def stream_list(self, path = None, layout = None, extended = False):
|
||||
"""Return a sorted list of [path, layout] lists. If 'path' or
|
||||
'layout' are specified, only return streams that match those
|
||||
exact values. If 'extended' is True, the returned lists have
|
||||
|
|
@ -71,11 +71,11 @@ class Client():
|
|||
if extended:
|
||||
params["extended"] = 1
|
||||
streams = self.http.get("stream/list", params)
|
||||
return nilmdb.utils.sort.sort_human(streams, key=lambda s: s[0])
|
||||
return nilmdb.utils.sort.sort_human(streams, key = lambda s: s[0])
|
||||
|
||||
def stream_get_metadata(self, path, keys=None):
|
||||
def stream_get_metadata(self, path, keys = None):
|
||||
"""Get stream metadata"""
|
||||
params = {"path": path}
|
||||
params = { "path": path }
|
||||
if keys is not None:
|
||||
params["key"] = keys
|
||||
return self.http.get("stream/get_metadata", params)
|
||||
|
|
@ -99,28 +99,22 @@ class Client():
|
|||
|
||||
def stream_create(self, path, layout):
|
||||
"""Create a new stream"""
|
||||
params = {
|
||||
"path": path,
|
||||
"layout": layout
|
||||
}
|
||||
params = { "path": path,
|
||||
"layout" : layout }
|
||||
return self.http.post("stream/create", params)
|
||||
|
||||
def stream_destroy(self, path):
|
||||
"""Delete stream. Fails if any data is still present."""
|
||||
params = {
|
||||
"path": path
|
||||
}
|
||||
params = { "path": path }
|
||||
return self.http.post("stream/destroy", params)
|
||||
|
||||
def stream_rename(self, oldpath, newpath):
|
||||
"""Rename a stream."""
|
||||
params = {
|
||||
"oldpath": oldpath,
|
||||
"newpath": newpath
|
||||
}
|
||||
params = { "oldpath": oldpath,
|
||||
"newpath": newpath }
|
||||
return self.http.post("stream/rename", params)
|
||||
|
||||
def stream_remove(self, path, start=None, end=None):
|
||||
def stream_remove(self, path, start = None, end = None):
|
||||
"""Remove data from the specified time range"""
|
||||
params = {
|
||||
"path": path
|
||||
|
|
@ -135,7 +129,7 @@ class Client():
|
|||
return total
|
||||
|
||||
@contextlib.contextmanager
|
||||
def stream_insert_context(self, path, start=None, end=None):
|
||||
def stream_insert_context(self, path, start = None, end = None):
|
||||
"""Return a context manager that allows data to be efficiently
|
||||
inserted into a stream in a piecewise manner. Data is
|
||||
provided as ASCII lines, and is aggregated and sent to the
|
||||
|
|
@ -145,8 +139,8 @@ class Client():
|
|||
|
||||
Example:
|
||||
with client.stream_insert_context('/path', start, end) as ctx:
|
||||
ctx.insert('1234567890000000 1 2 3 4\\n')
|
||||
ctx.insert('1234567891000000 1 2 3 4\\n')
|
||||
ctx.insert('1234567890.0 1 2 3 4\\n')
|
||||
ctx.insert('1234567891.0 1 2 3 4\\n')
|
||||
|
||||
For more details, see help for nilmdb.client.client.StreamInserter
|
||||
|
||||
|
|
@ -158,21 +152,21 @@ class Client():
|
|||
ctx.finalize()
|
||||
ctx.destroy()
|
||||
|
||||
def stream_insert(self, path, data, start=None, end=None):
|
||||
def stream_insert(self, path, data, start = None, end = None):
|
||||
"""Insert rows of data into a stream. data should be a string
|
||||
or iterable that provides ASCII data that matches the database
|
||||
layout for path. Data is passed through stream_insert_context,
|
||||
so it will be broken into reasonably-sized chunks and
|
||||
start/end will be deduced if missing."""
|
||||
with self.stream_insert_context(path, start, end) as ctx:
|
||||
if isinstance(data, bytes):
|
||||
if isinstance(data, basestring):
|
||||
ctx.insert(data)
|
||||
else:
|
||||
for chunk in data:
|
||||
ctx.insert(chunk)
|
||||
return ctx.last_response
|
||||
|
||||
def stream_insert_block(self, path, data, start, end, binary=False):
|
||||
def stream_insert_block(self, path, data, start, end, binary = False):
|
||||
"""Insert a single fixed block of data into the stream. It is
|
||||
sent directly to the server in one block with no further
|
||||
processing.
|
||||
|
|
@ -187,9 +181,9 @@ class Client():
|
|||
}
|
||||
if binary:
|
||||
params["binary"] = 1
|
||||
return self.http.put("stream/insert", data, params)
|
||||
return self.http.put("stream/insert", data, params, binary = binary)
|
||||
|
||||
def stream_intervals(self, path, start=None, end=None, diffpath=None):
|
||||
def stream_intervals(self, path, start = None, end = None, diffpath = None):
|
||||
"""
|
||||
Return a generator that yields each stream interval.
|
||||
|
||||
|
|
@ -207,8 +201,8 @@ class Client():
|
|||
params["end"] = timestamp_to_string(end)
|
||||
return self.http.get_gen("stream/intervals", params)
|
||||
|
||||
def stream_extract(self, path, start=None, end=None,
|
||||
count=False, markup=False, binary=False):
|
||||
def stream_extract(self, path, start = None, end = None,
|
||||
count = False, markup = False, binary = False):
|
||||
"""
|
||||
Extract data from a stream. Returns a generator that yields
|
||||
lines of ASCII-formatted data that matches the database
|
||||
|
|
@ -238,18 +232,17 @@ class Client():
|
|||
params["markup"] = 1
|
||||
if binary:
|
||||
params["binary"] = 1
|
||||
return self.http.get_gen("stream/extract", params, binary=binary)
|
||||
return self.http.get_gen("stream/extract", params, binary = binary)
|
||||
|
||||
def stream_count(self, path, start=None, end=None):
|
||||
def stream_count(self, path, start = None, end = None):
|
||||
"""
|
||||
Return the number of rows of data in the stream that satisfy
|
||||
the given timestamps.
|
||||
"""
|
||||
counts = list(self.stream_extract(path, start, end, count=True))
|
||||
counts = list(self.stream_extract(path, start, end, count = True))
|
||||
return int(counts[0])
|
||||
|
||||
|
||||
class StreamInserter():
|
||||
class StreamInserter(object):
|
||||
"""Object returned by stream_insert_context() that manages
|
||||
the insertion of rows of data into a particular path.
|
||||
|
||||
|
|
@ -337,8 +330,8 @@ class StreamInserter():
|
|||
|
||||
# Send the block once we have enough data
|
||||
if self._block_len >= maxdata:
|
||||
self._send_block(final=False)
|
||||
if self._block_len >= self._max_data_after_send:
|
||||
self._send_block(final = False)
|
||||
if self._block_len >= self._max_data_after_send: # pragma: no cover
|
||||
raise ValueError("too much data left over after trying"
|
||||
" to send intermediate block; is it"
|
||||
" missing newlines or malformed?")
|
||||
|
|
@ -364,12 +357,12 @@ class StreamInserter():
|
|||
|
||||
If more data is inserted after a finalize(), it will become
|
||||
part of a new interval and there may be a gap left in-between."""
|
||||
self._send_block(final=True)
|
||||
self._send_block(final = True)
|
||||
|
||||
def send(self):
|
||||
"""Send any data that we might have buffered up. Does not affect
|
||||
any other treatment of timestamps or endpoints."""
|
||||
self._send_block(final=False)
|
||||
self._send_block(final = False)
|
||||
|
||||
def _get_first_noncomment(self, block):
|
||||
"""Return the (start, end) indices of the first full line in
|
||||
|
|
@ -377,10 +370,10 @@ class StreamInserter():
|
|||
there isn't one."""
|
||||
start = 0
|
||||
while True:
|
||||
end = block.find(b'\n', start)
|
||||
end = block.find('\n', start)
|
||||
if end < 0:
|
||||
raise IndexError
|
||||
if block[start] != b'#'[0]:
|
||||
if block[start] != '#':
|
||||
return (start, (end + 1))
|
||||
start = end + 1
|
||||
|
||||
|
|
@ -388,22 +381,22 @@ class StreamInserter():
|
|||
"""Return the (start, end) indices of the last full line in
|
||||
block[:length] that isn't a comment, or raise IndexError if
|
||||
there isn't one."""
|
||||
end = block.rfind(b'\n')
|
||||
end = block.rfind('\n')
|
||||
if end <= 0:
|
||||
raise IndexError
|
||||
while True:
|
||||
start = block.rfind(b'\n', 0, end)
|
||||
if block[start + 1] != b'#'[0]:
|
||||
start = block.rfind('\n', 0, end)
|
||||
if block[start + 1] != '#':
|
||||
return ((start + 1), end)
|
||||
if start == -1:
|
||||
raise IndexError
|
||||
end = start
|
||||
|
||||
def _send_block(self, final=False):
|
||||
def _send_block(self, final = False):
|
||||
"""Send data currently in the block. The data sent will
|
||||
consist of full lines only, so some might be left over."""
|
||||
# Build the full string to send
|
||||
block = b"".join(self._block_data)
|
||||
block = "".join(self._block_data)
|
||||
|
||||
start_ts = self._interval_start
|
||||
if start_ts is None:
|
||||
|
|
@ -420,7 +413,7 @@ class StreamInserter():
|
|||
# or the timestamp of the last line plus epsilon.
|
||||
end_ts = self._interval_end
|
||||
try:
|
||||
if block[-1] != b'\n'[0]:
|
||||
if block[-1] != '\n':
|
||||
raise ValueError("final block didn't end with a newline")
|
||||
if end_ts is None:
|
||||
(spos, epos) = self._get_last_noncomment(block)
|
||||
|
|
@ -454,7 +447,7 @@ class StreamInserter():
|
|||
# the server complain so that the error is the same
|
||||
# as if we hadn't done this chunking.
|
||||
end_ts = self._interval_end
|
||||
self._block_data = [block[spos:]]
|
||||
self._block_data = [ block[spos:] ]
|
||||
self._block_len = (epos - spos)
|
||||
block = block[:spos]
|
||||
|
||||
|
|
@ -472,6 +465,6 @@ class StreamInserter():
|
|||
|
||||
# Send it
|
||||
self.last_response = self._client.stream_insert_block(
|
||||
self._path, block, start_ts, end_ts, binary=False)
|
||||
self._path, block, start_ts, end_ts, binary = False)
|
||||
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,41 +1,33 @@
|
|||
"""HTTP client errors"""
|
||||
|
||||
from nilmdb.utils.printf import sprintf
|
||||
|
||||
from nilmdb.utils.printf import *
|
||||
|
||||
class Error(Exception):
|
||||
"""Base exception for both ClientError and ServerError responses"""
|
||||
def __init__(self,
|
||||
status="Unspecified error",
|
||||
message=None,
|
||||
url=None,
|
||||
traceback=None):
|
||||
super().__init__(status)
|
||||
status = "Unspecified error",
|
||||
message = None,
|
||||
url = None,
|
||||
traceback = None):
|
||||
Exception.__init__(self, status)
|
||||
self.status = status # e.g. "400 Bad Request"
|
||||
self.message = message # textual message from the server
|
||||
self.url = url # URL we were requesting
|
||||
self.traceback = traceback # server traceback, if available
|
||||
|
||||
def _format_error(self, show_url):
|
||||
s = sprintf("[%s]", self.status)
|
||||
if self.message:
|
||||
s += sprintf(" %s", self.message)
|
||||
if show_url and self.url:
|
||||
if show_url and self.url: # pragma: no cover
|
||||
s += sprintf(" (%s)", self.url)
|
||||
if self.traceback:
|
||||
if self.traceback: # pragma: no cover
|
||||
s += sprintf("\nServer traceback:\n%s", self.traceback)
|
||||
return s
|
||||
|
||||
def __str__(self):
|
||||
return self._format_error(show_url=False)
|
||||
|
||||
def __repr__(self):
|
||||
return self._format_error(show_url=True)
|
||||
|
||||
|
||||
return self._format_error(show_url = False)
|
||||
def __repr__(self): # pragma: no cover
|
||||
return self._format_error(show_url = True)
|
||||
class ClientError(Error):
|
||||
pass
|
||||
|
||||
|
||||
class ServerError(Error):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,21 +1,21 @@
|
|||
"""HTTP client library"""
|
||||
|
||||
import json
|
||||
import urllib.parse
|
||||
import requests
|
||||
|
||||
import nilmdb.utils
|
||||
from nilmdb.client.errors import ClientError, ServerError, Error
|
||||
|
||||
import simplejson as json
|
||||
import urlparse
|
||||
import requests
|
||||
|
||||
class HTTPClient():
|
||||
class HTTPClient(object):
|
||||
"""Class to manage and perform HTTP requests from the client"""
|
||||
def __init__(self, baseurl="", post_json=False, verify_ssl=True):
|
||||
def __init__(self, baseurl = "", post_json = False, verify_ssl = True):
|
||||
"""If baseurl is supplied, all other functions that take
|
||||
a URL can be given a relative URL instead."""
|
||||
# Verify / clean up URL
|
||||
reparsed = urllib.parse.urlparse(baseurl).geturl()
|
||||
reparsed = urlparse.urlparse(baseurl).geturl()
|
||||
if '://' not in reparsed:
|
||||
reparsed = urllib.parse.urlparse("http://" + baseurl).geturl()
|
||||
reparsed = urlparse.urlparse("http://" + baseurl).geturl()
|
||||
self.baseurl = reparsed.rstrip('/') + '/'
|
||||
|
||||
# Note whether we want SSL verification
|
||||
|
|
@ -32,27 +32,25 @@ class HTTPClient():
|
|||
# Default variables for exception. We use the entire body as
|
||||
# the default message, in case we can't extract it from a JSON
|
||||
# response.
|
||||
args = {
|
||||
"url": url,
|
||||
"status": str(code),
|
||||
"message": body,
|
||||
"traceback": None
|
||||
}
|
||||
args = { "url" : url,
|
||||
"status" : str(code),
|
||||
"message" : body,
|
||||
"traceback" : None }
|
||||
try:
|
||||
# Fill with server-provided data if we can
|
||||
jsonerror = json.loads(body)
|
||||
args["status"] = jsonerror["status"]
|
||||
args["message"] = jsonerror["message"]
|
||||
args["traceback"] = jsonerror["traceback"]
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
if 400 <= code <= 499:
|
||||
if code >= 400 and code <= 499:
|
||||
raise ClientError(**args)
|
||||
else:
|
||||
if 500 <= code <= 599:
|
||||
else: # pragma: no cover
|
||||
if code >= 500 and code <= 599:
|
||||
if args["message"] is None:
|
||||
args["message"] = ("(no message; try disabling "
|
||||
"response.stream option in "
|
||||
args["message"] = ("(no message; try disabling " +
|
||||
"response.stream option in " +
|
||||
"nilmdb.server for better debugging)")
|
||||
raise ServerError(**args)
|
||||
else:
|
||||
|
|
@ -62,7 +60,7 @@ class HTTPClient():
|
|||
pass
|
||||
|
||||
def _do_req(self, method, url, query_data, body_data, stream, headers):
|
||||
url = urllib.parse.urljoin(self.baseurl, url)
|
||||
url = urlparse.urljoin(self.baseurl, url)
|
||||
try:
|
||||
# Create a new session, ensure we send "Connection: close",
|
||||
# and explicitly close connection after the transfer.
|
||||
|
|
@ -76,11 +74,11 @@ class HTTPClient():
|
|||
headers = {}
|
||||
headers["Connection"] = "close"
|
||||
response = session.request(method, url,
|
||||
params=query_data,
|
||||
data=body_data,
|
||||
stream=stream,
|
||||
headers=headers,
|
||||
verify=self.verify_ssl)
|
||||
params = query_data,
|
||||
data = body_data,
|
||||
stream = stream,
|
||||
headers = headers,
|
||||
verify = self.verify_ssl)
|
||||
|
||||
# Close the connection. If it's a generator (stream =
|
||||
# True), the requests library shouldn't actually close the
|
||||
|
|
@ -88,8 +86,8 @@ class HTTPClient():
|
|||
# response.
|
||||
session.close()
|
||||
except requests.RequestException as e:
|
||||
raise ServerError(status="502 Error", url=url,
|
||||
message=str(e))
|
||||
raise ServerError(status = "502 Error", url = url,
|
||||
message = str(e.message))
|
||||
if response.status_code != 200:
|
||||
self._handle_error(url, response.status_code, response.content)
|
||||
self._last_response = response
|
||||
|
|
@ -100,46 +98,48 @@ class HTTPClient():
|
|||
return (response, False)
|
||||
|
||||
# Normal versions that return data directly
|
||||
def _req(self, method, url, query=None, body=None, headers=None):
|
||||
def _req(self, method, url, query = None, body = None, headers = None):
|
||||
"""
|
||||
Make a request and return the body data as a string or parsed
|
||||
JSON object, or raise an error if it contained an error.
|
||||
"""
|
||||
(response, isjson) = self._do_req(method, url, query, body,
|
||||
stream=False, headers=headers)
|
||||
stream = False, headers = headers)
|
||||
if isjson:
|
||||
return json.loads(response.content)
|
||||
return response.text
|
||||
return response.content
|
||||
|
||||
def get(self, url, params=None):
|
||||
def get(self, url, params = None):
|
||||
"""Simple GET (parameters in URL)"""
|
||||
return self._req("GET", url, params, None)
|
||||
|
||||
def post(self, url, params=None):
|
||||
def post(self, url, params = None):
|
||||
"""Simple POST (parameters in body)"""
|
||||
if self.post_json:
|
||||
return self._req("POST", url, None,
|
||||
json.dumps(params),
|
||||
{'Content-type': 'application/json'})
|
||||
{ 'Content-type': 'application/json' })
|
||||
else:
|
||||
return self._req("POST", url, None, params)
|
||||
|
||||
def put(self, url, data, params=None,
|
||||
content_type="application/octet-stream"):
|
||||
def put(self, url, data, params = None, binary = False):
|
||||
"""Simple PUT (parameters in URL, data in body)"""
|
||||
h = {'Content-type': content_type}
|
||||
return self._req("PUT", url, query=params, body=data, headers=h)
|
||||
if binary:
|
||||
h = { 'Content-type': 'application/octet-stream' }
|
||||
else:
|
||||
h = { 'Content-type': 'text/plain; charset=utf-8' }
|
||||
return self._req("PUT", url, query = params, body = data, headers = h)
|
||||
|
||||
# Generator versions that return data one line at a time.
|
||||
def _req_gen(self, method, url, query=None, body=None,
|
||||
headers=None, binary=False):
|
||||
def _req_gen(self, method, url, query = None, body = None,
|
||||
headers = None, binary = False):
|
||||
"""
|
||||
Make a request and return a generator that gives back strings
|
||||
or JSON decoded lines of the body data, or raise an error if
|
||||
it contained an eror.
|
||||
"""
|
||||
(response, isjson) = self._do_req(method, url, query, body,
|
||||
stream=True, headers=headers)
|
||||
stream = True, headers = headers)
|
||||
|
||||
# Like the iter_lines function in Requests, but only splits on
|
||||
# the specified line ending.
|
||||
|
|
@ -156,32 +156,32 @@ class HTTPClient():
|
|||
pending = tmp[-1]
|
||||
for line in lines:
|
||||
yield line
|
||||
if pending is not None:
|
||||
if pending is not None: # pragma: no cover (missing newline)
|
||||
yield pending
|
||||
|
||||
# Yield the chunks or lines as requested
|
||||
if binary:
|
||||
for chunk in response.iter_content(chunk_size=65536):
|
||||
for chunk in response.iter_content(chunk_size = 65536):
|
||||
yield chunk
|
||||
elif isjson:
|
||||
for line in lines(response.iter_content(chunk_size=1),
|
||||
ending=b'\r\n'):
|
||||
for line in lines(response.iter_content(chunk_size = 1),
|
||||
ending = '\r\n'):
|
||||
yield json.loads(line)
|
||||
else:
|
||||
for line in lines(response.iter_content(chunk_size=65536),
|
||||
ending=b'\n'):
|
||||
for line in lines(response.iter_content(chunk_size = 65536),
|
||||
ending = '\n'):
|
||||
yield line
|
||||
|
||||
def get_gen(self, url, params=None, binary=False):
|
||||
def get_gen(self, url, params = None, binary = False):
|
||||
"""Simple GET (parameters in URL) returning a generator"""
|
||||
return self._req_gen("GET", url, params, binary=binary)
|
||||
return self._req_gen("GET", url, params, binary = binary)
|
||||
|
||||
def post_gen(self, url, params=None):
|
||||
def post_gen(self, url, params = None):
|
||||
"""Simple POST (parameters in body) returning a generator"""
|
||||
if self.post_json:
|
||||
return self._req_gen("POST", url, None,
|
||||
json.dumps(params),
|
||||
{'Content-type': 'application/json'})
|
||||
{ 'Content-type': 'application/json' })
|
||||
else:
|
||||
return self._req_gen("POST", url, None, params)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,33 +3,29 @@
|
|||
"""Provide a NumpyClient class that is based on normal Client, but has
|
||||
additional methods for extracting and inserting data via Numpy arrays."""
|
||||
|
||||
import contextlib
|
||||
|
||||
import numpy
|
||||
|
||||
import nilmdb.utils
|
||||
import nilmdb.client.client
|
||||
import nilmdb.client.httpclient
|
||||
from nilmdb.client.errors import ClientError
|
||||
|
||||
import contextlib
|
||||
from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
|
||||
|
||||
import numpy
|
||||
import cStringIO
|
||||
|
||||
def layout_to_dtype(layout):
|
||||
ltype = layout.split('_')[0]
|
||||
lcount = int(layout.split('_')[1])
|
||||
if ltype.startswith('int'):
|
||||
atype = '<i' + str(int(ltype[3:]) // 8)
|
||||
atype = '<i' + str(int(ltype[3:]) / 8)
|
||||
elif ltype.startswith('uint'):
|
||||
atype = '<u' + str(int(ltype[4:]) // 8)
|
||||
atype = '<u' + str(int(ltype[4:]) / 8)
|
||||
elif ltype.startswith('float'):
|
||||
atype = '<f' + str(int(ltype[5:]) // 8)
|
||||
atype = '<f' + str(int(ltype[5:]) / 8)
|
||||
else:
|
||||
raise ValueError("bad layout")
|
||||
if lcount == 1:
|
||||
dtype = [('timestamp', '<i8'), ('data', atype)]
|
||||
else:
|
||||
dtype = [('timestamp', '<i8'), ('data', atype, lcount)]
|
||||
return numpy.dtype(dtype)
|
||||
|
||||
return numpy.dtype([('timestamp', '<i8'), ('data', atype, lcount)])
|
||||
|
||||
class NumpyClient(nilmdb.client.client.Client):
|
||||
"""Subclass of nilmdb.client.Client that adds additional methods for
|
||||
|
|
@ -43,9 +39,9 @@ class NumpyClient(nilmdb.client.client.Client):
|
|||
layout = streams[0][1]
|
||||
return layout_to_dtype(layout)
|
||||
|
||||
def stream_extract_numpy(self, path, start=None, end=None,
|
||||
layout=None, maxrows=100000,
|
||||
structured=False):
|
||||
def stream_extract_numpy(self, path, start = None, end = None,
|
||||
layout = None, maxrows = 100000,
|
||||
structured = False):
|
||||
"""
|
||||
Extract data from a stream. Returns a generator that yields
|
||||
Numpy arrays of up to 'maxrows' of data each.
|
||||
|
|
@ -59,7 +55,7 @@ class NumpyClient(nilmdb.client.client.Client):
|
|||
dtype = self._get_dtype(path, layout)
|
||||
|
||||
def to_numpy(data):
|
||||
a = numpy.frombuffer(data, dtype)
|
||||
a = numpy.fromstring(data, dtype)
|
||||
if structured:
|
||||
return a
|
||||
return numpy.c_[a['timestamp'], a['data']]
|
||||
|
|
@ -67,25 +63,25 @@ class NumpyClient(nilmdb.client.client.Client):
|
|||
chunks = []
|
||||
total_len = 0
|
||||
maxsize = dtype.itemsize * maxrows
|
||||
for data in self.stream_extract(path, start, end, binary=True):
|
||||
for data in self.stream_extract(path, start, end, binary = True):
|
||||
# Add this block of binary data
|
||||
chunks.append(data)
|
||||
total_len += len(data)
|
||||
|
||||
# See if we have enough to make the requested Numpy array
|
||||
while total_len >= maxsize:
|
||||
assembled = b"".join(chunks)
|
||||
assembled = "".join(chunks)
|
||||
total_len -= maxsize
|
||||
chunks = [assembled[maxsize:]]
|
||||
chunks = [ assembled[maxsize:] ]
|
||||
block = assembled[:maxsize]
|
||||
yield to_numpy(block)
|
||||
|
||||
if total_len:
|
||||
yield to_numpy(b"".join(chunks))
|
||||
yield to_numpy("".join(chunks))
|
||||
|
||||
@contextlib.contextmanager
|
||||
def stream_insert_numpy_context(self, path, start=None, end=None,
|
||||
layout=None):
|
||||
def stream_insert_numpy_context(self, path, start = None, end = None,
|
||||
layout = None):
|
||||
"""Return a context manager that allows data to be efficiently
|
||||
inserted into a stream in a piecewise manner. Data is
|
||||
provided as Numpy arrays, and is aggregated and sent to the
|
||||
|
|
@ -104,8 +100,8 @@ class NumpyClient(nilmdb.client.client.Client):
|
|||
ctx.finalize()
|
||||
ctx.destroy()
|
||||
|
||||
def stream_insert_numpy(self, path, data, start=None, end=None,
|
||||
layout=None):
|
||||
def stream_insert_numpy(self, path, data, start = None, end = None,
|
||||
layout = None):
|
||||
"""Insert data into a stream. data should be a Numpy array
|
||||
which will be passed through stream_insert_numpy_context to
|
||||
break it into chunks etc. See the help for that function
|
||||
|
|
@ -118,7 +114,6 @@ class NumpyClient(nilmdb.client.client.Client):
|
|||
ctx.insert(chunk)
|
||||
return ctx.last_response
|
||||
|
||||
|
||||
class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
|
||||
"""Object returned by stream_insert_numpy_context() that manages
|
||||
the insertion of rows of data into a particular path.
|
||||
|
|
@ -151,7 +146,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
|
|||
|
||||
def insert(self, array):
|
||||
"""Insert Numpy data, which must match the layout type."""
|
||||
if not isinstance(array, numpy.ndarray):
|
||||
if type(array) != numpy.ndarray:
|
||||
array = numpy.array(array)
|
||||
if array.ndim == 1:
|
||||
# Already a structured array; just verify the type
|
||||
|
|
@ -161,9 +156,9 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
|
|||
# Convert to structured array
|
||||
sarray = numpy.zeros(array.shape[0], dtype=self._dtype)
|
||||
try:
|
||||
sarray['timestamp'] = array[:, 0]
|
||||
sarray['timestamp'] = array[:,0]
|
||||
# Need the squeeze in case sarray['data'] is 1 dimensional
|
||||
sarray['data'] = numpy.squeeze(array[:, 1:])
|
||||
sarray['data'] = numpy.squeeze(array[:,1:])
|
||||
except (IndexError, ValueError):
|
||||
raise ValueError("wrong number of fields for this data type")
|
||||
array = sarray
|
||||
|
|
@ -189,15 +184,15 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
|
|||
|
||||
# Send if it's too long
|
||||
if self._block_rows >= maxrows:
|
||||
self._send_block(final=False)
|
||||
self._send_block(final = False)
|
||||
|
||||
def _send_block(self, final=False):
|
||||
def _send_block(self, final = False):
|
||||
"""Send the data current stored up. One row might be left
|
||||
over if we need its timestamp saved."""
|
||||
|
||||
# Build the full array to send
|
||||
if self._block_rows == 0:
|
||||
array = numpy.zeros(0, dtype=self._dtype)
|
||||
array = numpy.zeros(0, dtype = self._dtype)
|
||||
else:
|
||||
array = numpy.hstack(self._block_arrays)
|
||||
|
||||
|
|
@ -241,7 +236,7 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
|
|||
# the server complain so that the error is the same
|
||||
# as if we hadn't done this chunking.
|
||||
end_ts = self._interval_end
|
||||
self._block_arrays = [array[-1:]]
|
||||
self._block_arrays = [ array[-1:] ]
|
||||
self._block_rows = 1
|
||||
array = array[:-1]
|
||||
|
||||
|
|
@ -251,13 +246,13 @@ class StreamInserterNumpy(nilmdb.client.client.StreamInserter):
|
|||
# If we have no endpoints, or equal endpoints, it's OK as long
|
||||
# as there's no data to send
|
||||
if (start_ts is None or end_ts is None) or (start_ts == end_ts):
|
||||
if not array:
|
||||
if len(array) == 0:
|
||||
return
|
||||
raise ClientError("have data to send, but invalid start/end times")
|
||||
|
||||
# Send it
|
||||
data = array.tostring()
|
||||
self.last_response = self._client.stream_insert_block(
|
||||
self._path, data, start_ts, end_ts, binary=True)
|
||||
self._path, data, start_ts, end_ts, binary = True)
|
||||
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,28 +1,31 @@
|
|||
"""Command line client functionality"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
import argparse
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
import nilmdb.client
|
||||
from nilmdb.utils.printf import fprintf, sprintf
|
||||
|
||||
from nilmdb.utils.printf import *
|
||||
from nilmdb.utils import datetime_tz
|
||||
import nilmdb.utils.time
|
||||
|
||||
import argcomplete
|
||||
import datetime_tz
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
import signal
|
||||
|
||||
try: # pragma: no cover
|
||||
import argcomplete
|
||||
except ImportError: # pragma: no cover
|
||||
argcomplete = None
|
||||
|
||||
# Valid subcommands. Defined in separate files just to break
|
||||
# things up -- they're still called with Cmdline as self.
|
||||
subcommands = ["help", "info", "create", "rename", "list", "intervals",
|
||||
"metadata", "insert", "extract", "remove", "destroy"]
|
||||
subcommands = [ "help", "info", "create", "rename", "list", "intervals",
|
||||
"metadata", "insert", "extract", "remove", "destroy" ]
|
||||
|
||||
# Import the subcommand modules
|
||||
subcmd_mods = {}
|
||||
for cmd in subcommands:
|
||||
subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist=[cmd])
|
||||
|
||||
subcmd_mods[cmd] = __import__("nilmdb.cmdline." + cmd, fromlist = [ cmd ])
|
||||
|
||||
class JimArgumentParser(argparse.ArgumentParser):
|
||||
def parse_args(self, args=None, namespace=None):
|
||||
|
|
@ -30,19 +33,18 @@ class JimArgumentParser(argparse.ArgumentParser):
|
|||
# --version". This makes "nilmtool cmd --version" work, which
|
||||
# is needed by help2man.
|
||||
if "--version" in (args or sys.argv[1:]):
|
||||
args = ["--version"]
|
||||
args = [ "--version" ]
|
||||
return argparse.ArgumentParser.parse_args(self, args, namespace)
|
||||
|
||||
def error(self, message):
|
||||
self.print_usage(sys.stderr)
|
||||
self.exit(2, sprintf("error: %s\n", message))
|
||||
|
||||
|
||||
class Complete():
|
||||
class Complete(object): # pragma: no cover
|
||||
# Completion helpers, for using argcomplete (see
|
||||
# extras/nilmtool-bash-completion.sh)
|
||||
def escape(self, s):
|
||||
quote_chars = ["\\", "\"", "'", " "]
|
||||
quote_chars = [ "\\", "\"", "'", " " ]
|
||||
for char in quote_chars:
|
||||
s = s.replace(char, "\\" + char)
|
||||
return s
|
||||
|
|
@ -55,18 +57,18 @@ class Complete():
|
|||
|
||||
def path(self, prefix, parsed_args, **kwargs):
|
||||
client = nilmdb.client.Client(parsed_args.url)
|
||||
return (self.escape(s[0])
|
||||
return ( self.escape(s[0])
|
||||
for s in client.stream_list()
|
||||
if s[0].startswith(prefix))
|
||||
if s[0].startswith(prefix) )
|
||||
|
||||
def layout(self, prefix, parsed_args, **kwargs):
|
||||
types = ["int8", "int16", "int32", "int64",
|
||||
types = [ "int8", "int16", "int32", "int64",
|
||||
"uint8", "uint16", "uint32", "uint64",
|
||||
"float32", "float64"]
|
||||
"float32", "float64" ]
|
||||
layouts = []
|
||||
for i in range(1, 10):
|
||||
for i in range(1,10):
|
||||
layouts.extend([(t + "_" + str(i)) for t in types])
|
||||
return (lay for lay in layouts if lay.startswith(prefix))
|
||||
return ( l for l in layouts if l.startswith(prefix) )
|
||||
|
||||
def meta_key(self, prefix, parsed_args, **kwargs):
|
||||
return (kv.split('=')[0] for kv
|
||||
|
|
@ -78,22 +80,30 @@ class Complete():
|
|||
if not path:
|
||||
return []
|
||||
results = []
|
||||
for (k, v) in client.stream_get_metadata(path).items():
|
||||
# prefix comes in as UTF-8, but results need to be Unicode,
|
||||
# weird. Still doesn't work in all cases, but that's bugs in
|
||||
# argcomplete.
|
||||
prefix = nilmdb.utils.unicode.decode(prefix)
|
||||
for (k,v) in client.stream_get_metadata(path).iteritems():
|
||||
kv = self.escape(k + '=' + v)
|
||||
if kv.startswith(prefix):
|
||||
results.append(kv)
|
||||
return results
|
||||
|
||||
class Cmdline(object):
|
||||
|
||||
class Cmdline():
|
||||
|
||||
def __init__(self, argv=None):
|
||||
def __init__(self, argv = None):
|
||||
self.argv = argv or sys.argv[1:]
|
||||
try:
|
||||
# Assume command line arguments are encoded with stdin's encoding,
|
||||
# and reverse it. Won't be needed in Python 3, but for now..
|
||||
self.argv = [ x.decode(sys.stdin.encoding) for x in self.argv ]
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
self.client = None
|
||||
self.def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
|
||||
self.subcmd = {}
|
||||
self.complete = Complete()
|
||||
self.complete_output_stream = None # overridden by test suite
|
||||
|
||||
def arg_time(self, toparse):
|
||||
"""Parse a time string argument"""
|
||||
|
|
@ -105,14 +115,14 @@ class Cmdline():
|
|||
|
||||
# Set up the parser
|
||||
def parser_setup(self):
|
||||
self.parser = JimArgumentParser(add_help=False,
|
||||
formatter_class=def_form)
|
||||
self.parser = JimArgumentParser(add_help = False,
|
||||
formatter_class = def_form)
|
||||
|
||||
group = self.parser.add_argument_group("General options")
|
||||
group.add_argument("-h", "--help", action='help',
|
||||
help='show this help message and exit')
|
||||
group.add_argument("-v", "--version", action="version",
|
||||
version=nilmdb.__version__)
|
||||
version = nilmdb.__version__)
|
||||
|
||||
group = self.parser.add_argument_group("Server")
|
||||
group.add_argument("-u", "--url", action="store",
|
||||
|
|
@ -121,7 +131,7 @@ class Cmdline():
|
|||
).completer = self.complete.url
|
||||
|
||||
sub = self.parser.add_subparsers(
|
||||
title="Commands", dest="command", required=True,
|
||||
title="Commands", dest="command",
|
||||
description="Use 'help command' or 'command --help' for more "
|
||||
"details on a particular command.")
|
||||
|
||||
|
|
@ -138,7 +148,10 @@ class Cmdline():
|
|||
def run(self):
|
||||
# Set SIGPIPE to its default handler -- we don't need Python
|
||||
# to catch it for us.
|
||||
try:
|
||||
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
||||
except ValueError: # pragma: no cover
|
||||
pass
|
||||
|
||||
# Clear cached timezone, so that we can pick up timezone changes
|
||||
# while running this from the test suite.
|
||||
|
|
@ -146,8 +159,8 @@ class Cmdline():
|
|||
|
||||
# Run parser
|
||||
self.parser_setup()
|
||||
argcomplete.autocomplete(self.parser, exit_method=sys.exit,
|
||||
output_stream=self.complete_output_stream)
|
||||
if argcomplete: # pragma: no cover
|
||||
argcomplete.autocomplete(self.parser)
|
||||
self.args = self.parser.parse_args(self.argv)
|
||||
|
||||
# Run arg verify handler if there is one
|
||||
|
|
@ -160,7 +173,7 @@ class Cmdline():
|
|||
# unless the particular command requests that we don't.
|
||||
if "no_test_connect" not in self.args:
|
||||
try:
|
||||
self.client.version()
|
||||
server_version = self.client.version()
|
||||
except nilmdb.client.Error as e:
|
||||
self.die("error connecting to server: %s", str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
from argparse import RawDescriptionHelpFormatter as raw_form
|
||||
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.client
|
||||
|
||||
from argparse import RawDescriptionHelpFormatter as raw_form
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("create", help="Create a new stream",
|
||||
formatter_class=raw_form,
|
||||
formatter_class = raw_form,
|
||||
description="""
|
||||
Create a new empty stream at the specified path and with the specified
|
||||
layout type.
|
||||
|
|
@ -19,7 +19,7 @@ Layout types are of the format: type_count
|
|||
For example, 'float32_8' means the data for this stream has 8 columns of
|
||||
32-bit floating point values.
|
||||
""")
|
||||
cmd.set_defaults(handler=cmd_create)
|
||||
cmd.set_defaults(handler = cmd_create)
|
||||
group = cmd.add_argument_group("Required arguments")
|
||||
group.add_argument("path",
|
||||
help="Path (in database) of new stream, e.g. /foo/bar",
|
||||
|
|
@ -29,7 +29,6 @@ Layout types are of the format: type_count
|
|||
).completer = self.complete.layout
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_create(self):
|
||||
"""Create new stream"""
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
from nilmdb.utils.printf import *
|
||||
import nilmdb.client
|
||||
import fnmatch
|
||||
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
from nilmdb.utils.printf import printf
|
||||
import nilmdb.client
|
||||
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("destroy", help="Delete a stream and all data",
|
||||
formatter_class=def_form,
|
||||
formatter_class = def_form,
|
||||
description="""
|
||||
Destroy the stream at the specified path.
|
||||
The stream must be empty. All metadata
|
||||
|
|
@ -16,7 +14,7 @@ def setup(self, sub):
|
|||
|
||||
Wildcards and multiple paths are supported.
|
||||
""")
|
||||
cmd.set_defaults(handler=cmd_destroy)
|
||||
cmd.set_defaults(handler = cmd_destroy)
|
||||
group = cmd.add_argument_group("Options")
|
||||
group.add_argument("-R", "--remove", action="store_true",
|
||||
help="Remove all data before destroying stream")
|
||||
|
|
@ -29,10 +27,9 @@ def setup(self, sub):
|
|||
).completer = self.complete.path
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_destroy(self):
|
||||
"""Destroy stream"""
|
||||
streams = [s[0] for s in self.client.stream_list()]
|
||||
streams = [ s[0] for s in self.client.stream_list() ]
|
||||
paths = []
|
||||
for path in self.args.path:
|
||||
new = fnmatch.filter(streams, path)
|
||||
|
|
@ -46,7 +43,7 @@ def cmd_destroy(self):
|
|||
|
||||
try:
|
||||
if self.args.remove:
|
||||
self.client.stream_remove(path)
|
||||
count = self.client.stream_remove(path)
|
||||
self.client.stream_destroy(path)
|
||||
except nilmdb.client.ClientError as e:
|
||||
self.die("error destroying stream: %s", str(e))
|
||||
|
|
|
|||
|
|
@ -1,16 +1,15 @@
|
|||
import sys
|
||||
|
||||
from nilmdb.utils.printf import printf
|
||||
from __future__ import print_function
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.client
|
||||
|
||||
import sys
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("extract", help="Extract data",
|
||||
description="""
|
||||
Extract data from a stream.
|
||||
""")
|
||||
cmd.set_defaults(verify=cmd_extract_verify,
|
||||
handler=cmd_extract)
|
||||
cmd.set_defaults(verify = cmd_extract_verify,
|
||||
handler = cmd_extract)
|
||||
|
||||
group = cmd.add_argument_group("Data selection")
|
||||
group.add_argument("path",
|
||||
|
|
@ -41,8 +40,8 @@ def setup(self, sub):
|
|||
help="Just output a count of matched data points")
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_extract_verify(self):
|
||||
if self.args.start is not None and self.args.end is not None:
|
||||
if self.args.start > self.args.end:
|
||||
self.parser.error("start is after end")
|
||||
|
||||
|
|
@ -51,7 +50,6 @@ def cmd_extract_verify(self):
|
|||
self.args.timestamp_raw or self.args.count):
|
||||
self.parser.error("--binary cannot be combined with other options")
|
||||
|
||||
|
||||
def cmd_extract(self):
|
||||
streams = self.client.stream_list(self.args.path)
|
||||
if len(streams) != 1:
|
||||
|
|
@ -71,9 +69,9 @@ def cmd_extract(self):
|
|||
|
||||
printed = False
|
||||
if self.args.binary:
|
||||
printer = sys.stdout.buffer.write
|
||||
printer = sys.stdout.write
|
||||
else:
|
||||
printer = lambda x: print(x.decode('utf-8'))
|
||||
printer = print
|
||||
bare = self.args.bare
|
||||
count = self.args.count
|
||||
for dataline in self.client.stream_extract(self.args.path,
|
||||
|
|
@ -85,7 +83,7 @@ def cmd_extract(self):
|
|||
if bare and not count:
|
||||
# Strip timestamp (first element). Doesn't make sense
|
||||
# if we are only returning a count.
|
||||
dataline = b' '.join(dataline.split(b' ')[1:])
|
||||
dataline = ' '.join(dataline.split(' ')[1:])
|
||||
printer(dataline)
|
||||
printed = True
|
||||
if not printed:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import argparse
|
||||
from nilmdb.utils.printf import *
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("help", help="Show detailed help for a command",
|
||||
|
|
@ -7,15 +9,14 @@ def setup(self, sub):
|
|||
Show help for a command. 'help command' is
|
||||
the same as 'command --help'.
|
||||
""")
|
||||
cmd.set_defaults(handler=cmd_help)
|
||||
cmd.set_defaults(no_test_connect=True)
|
||||
cmd.set_defaults(handler = cmd_help)
|
||||
cmd.set_defaults(no_test_connect = True)
|
||||
cmd.add_argument("command", nargs="?",
|
||||
help="Command to get help about")
|
||||
cmd.add_argument("rest", nargs=argparse.REMAINDER,
|
||||
help=argparse.SUPPRESS)
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_help(self):
|
||||
if self.args.command in self.subcmd:
|
||||
self.subcmd[self.args.command].print_help()
|
||||
|
|
|
|||
|
|
@ -1,21 +1,19 @@
|
|||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
import nilmdb.client
|
||||
from nilmdb.utils.printf import printf
|
||||
from nilmdb.utils.printf import *
|
||||
from nilmdb.utils import human_size
|
||||
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("info", help="Server information",
|
||||
formatter_class=def_form,
|
||||
formatter_class = def_form,
|
||||
description="""
|
||||
List information about the server, like
|
||||
version.
|
||||
""")
|
||||
cmd.set_defaults(handler=cmd_info)
|
||||
cmd.set_defaults(handler = cmd_info)
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_info(self):
|
||||
"""Print info about the server"""
|
||||
printf("Client version: %s\n", nilmdb.__version__)
|
||||
|
|
|
|||
|
|
@ -1,18 +1,17 @@
|
|||
import sys
|
||||
|
||||
from nilmdb.utils.printf import printf
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.client
|
||||
import nilmdb.utils.timestamper as timestamper
|
||||
import nilmdb.utils.time
|
||||
|
||||
import sys
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("insert", help="Insert data",
|
||||
description="""
|
||||
Insert data into a stream.
|
||||
""")
|
||||
cmd.set_defaults(verify=cmd_insert_verify,
|
||||
handler=cmd_insert)
|
||||
cmd.set_defaults(verify = cmd_insert_verify,
|
||||
handler = cmd_insert)
|
||||
cmd.add_argument("-q", "--quiet", action='store_true',
|
||||
help='suppress unnecessary messages')
|
||||
|
||||
|
|
@ -62,24 +61,21 @@ def setup(self, sub):
|
|||
group.add_argument("path",
|
||||
help="Path of stream, e.g. /foo/bar",
|
||||
).completer = self.complete.path
|
||||
group.add_argument("file", nargs='?', default='-',
|
||||
group.add_argument("file", nargs = '?', default='-',
|
||||
help="File to insert (default: - (stdin))")
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_insert_verify(self):
|
||||
if self.args.timestamp:
|
||||
if not self.args.rate:
|
||||
self.die("error: --rate is needed, but was not specified")
|
||||
if not self.args.filename and self.args.start is None:
|
||||
self.die("error: need --start or --filename "
|
||||
"when adding timestamps")
|
||||
self.die("error: need --start or --filename when adding timestamps")
|
||||
else:
|
||||
if self.args.start is None or self.args.end is None:
|
||||
self.die("error: when not adding timestamps, --start and "
|
||||
"--end are required")
|
||||
|
||||
|
||||
def cmd_insert(self):
|
||||
# Find requested stream
|
||||
streams = self.client.stream_list(self.args.path)
|
||||
|
|
@ -91,7 +87,7 @@ def cmd_insert(self):
|
|||
try:
|
||||
filename = arg.file
|
||||
if filename == '-':
|
||||
infile = sys.stdin.buffer
|
||||
infile = sys.stdin
|
||||
else:
|
||||
try:
|
||||
infile = open(filename, "rb")
|
||||
|
|
@ -108,7 +104,7 @@ def cmd_insert(self):
|
|||
if arg.timestamp:
|
||||
data = timestamper.TimestamperRate(infile, arg.start, arg.rate)
|
||||
else:
|
||||
data = iter(lambda: infile.read(1048576), b'')
|
||||
data = iter(lambda: infile.read(1048576), '')
|
||||
|
||||
# Print info
|
||||
if not arg.quiet:
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
from nilmdb.utils.printf import printf
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.utils.time
|
||||
from nilmdb.utils.interval import Interval
|
||||
|
||||
import fnmatch
|
||||
import argparse
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("intervals", help="List intervals",
|
||||
formatter_class=def_form,
|
||||
formatter_class = def_form,
|
||||
description="""
|
||||
List intervals in a stream, similar to
|
||||
'list --detail path'.
|
||||
|
|
@ -16,8 +17,8 @@ def setup(self, sub):
|
|||
interval ranges that are present in 'path'
|
||||
and not present in 'diffpath' are printed.
|
||||
""")
|
||||
cmd.set_defaults(verify=cmd_intervals_verify,
|
||||
handler=cmd_intervals)
|
||||
cmd.set_defaults(verify = cmd_intervals_verify,
|
||||
handler = cmd_intervals)
|
||||
|
||||
group = cmd.add_argument_group("Stream selection")
|
||||
group.add_argument("path", metavar="PATH",
|
||||
|
|
@ -47,13 +48,11 @@ def setup(self, sub):
|
|||
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_intervals_verify(self):
|
||||
if self.args.start is not None and self.args.end is not None:
|
||||
if self.args.start >= self.args.end:
|
||||
self.parser.error("start must precede end")
|
||||
|
||||
|
||||
def cmd_intervals(self):
|
||||
"""List intervals in a stream"""
|
||||
if self.args.timestamp_raw:
|
||||
|
|
@ -62,11 +61,11 @@ def cmd_intervals(self):
|
|||
time_string = nilmdb.utils.time.timestamp_to_human
|
||||
|
||||
try:
|
||||
intervals = (Interval(start, end) for (start, end) in
|
||||
intervals = ( Interval(start, end) for (start, end) in
|
||||
self.client.stream_intervals(self.args.path,
|
||||
self.args.start,
|
||||
self.args.end,
|
||||
self.args.diff))
|
||||
self.args.diff) )
|
||||
if self.args.optimize:
|
||||
intervals = nilmdb.utils.interval.optimize(intervals)
|
||||
for i in intervals:
|
||||
|
|
@ -74,3 +73,4 @@ def cmd_intervals(self):
|
|||
|
||||
except nilmdb.client.ClientError as e:
|
||||
self.die("error listing intervals: %s", str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,21 +1,21 @@
|
|||
import fnmatch
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
from nilmdb.utils.printf import printf
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.utils.time
|
||||
|
||||
import fnmatch
|
||||
import argparse
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("list", help="List streams",
|
||||
formatter_class=def_form,
|
||||
formatter_class = def_form,
|
||||
description="""
|
||||
List streams available in the database,
|
||||
optionally filtering by path. Wildcards
|
||||
are accepted; non-matching paths or wildcards
|
||||
are ignored.
|
||||
""")
|
||||
cmd.set_defaults(verify=cmd_list_verify,
|
||||
handler=cmd_list)
|
||||
cmd.set_defaults(verify = cmd_list_verify,
|
||||
handler = cmd_list)
|
||||
|
||||
group = cmd.add_argument_group("Stream filtering")
|
||||
group.add_argument("path", metavar="PATH", default=["*"], nargs='*',
|
||||
|
|
@ -50,7 +50,6 @@ def setup(self, sub):
|
|||
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_list_verify(self):
|
||||
if self.args.start is not None and self.args.end is not None:
|
||||
if self.args.start >= self.args.end:
|
||||
|
|
@ -58,13 +57,11 @@ def cmd_list_verify(self):
|
|||
|
||||
if self.args.start is not None or self.args.end is not None:
|
||||
if not self.args.detail:
|
||||
self.parser.error("--start and --end only make sense "
|
||||
"with --detail")
|
||||
|
||||
self.parser.error("--start and --end only make sense with --detail")
|
||||
|
||||
def cmd_list(self):
|
||||
"""List available streams"""
|
||||
streams = self.client.stream_list(extended=True)
|
||||
streams = self.client.stream_list(extended = True)
|
||||
|
||||
if self.args.timestamp_raw:
|
||||
time_string = nilmdb.utils.time.timestamp_to_string
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
from nilmdb.utils.printf import printf
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb
|
||||
import nilmdb.client
|
||||
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("metadata", help="Get or set stream metadata",
|
||||
description="""
|
||||
|
|
@ -12,7 +11,7 @@ def setup(self, sub):
|
|||
usage="%(prog)s path [-g [key ...] | "
|
||||
"-s key=value [...] | -u key=value [...]] | "
|
||||
"-d [key ...]")
|
||||
cmd.set_defaults(handler=cmd_metadata)
|
||||
cmd.set_defaults(handler = cmd_metadata)
|
||||
|
||||
group = cmd.add_argument_group("Required arguments")
|
||||
group.add_argument("path",
|
||||
|
|
@ -37,16 +36,15 @@ def setup(self, sub):
|
|||
).completer = self.complete.meta_key
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_metadata(self):
|
||||
"""Manipulate metadata"""
|
||||
if self.args.set is not None or self.args.update is not None:
|
||||
# Either set, or update
|
||||
if self.args.set is not None:
|
||||
keyvals = self.args.set
|
||||
keyvals = map(nilmdb.utils.unicode.decode, self.args.set)
|
||||
handler = self.client.stream_set_metadata
|
||||
else:
|
||||
keyvals = self.args.update
|
||||
keyvals = map(nilmdb.utils.unicode.decode, self.args.update)
|
||||
handler = self.client.stream_update_metadata
|
||||
|
||||
# Extract key=value pairs
|
||||
|
|
@ -66,7 +64,7 @@ def cmd_metadata(self):
|
|||
# Delete (by setting values to empty strings)
|
||||
keys = None
|
||||
if self.args.delete:
|
||||
keys = list(self.args.delete)
|
||||
keys = map(nilmdb.utils.unicode.decode, self.args.delete)
|
||||
try:
|
||||
data = self.client.stream_get_metadata(self.args.path, keys)
|
||||
for key in data:
|
||||
|
|
@ -78,7 +76,7 @@ def cmd_metadata(self):
|
|||
# Get (or unspecified)
|
||||
keys = None
|
||||
if self.args.get:
|
||||
keys = list(self.args.get)
|
||||
keys = map(nilmdb.utils.unicode.decode, self.args.get)
|
||||
try:
|
||||
data = self.client.stream_get_metadata(self.args.path, keys)
|
||||
except nilmdb.client.ClientError as e:
|
||||
|
|
@ -87,4 +85,6 @@ def cmd_metadata(self):
|
|||
# Print nonexistant keys as having empty value
|
||||
if value is None:
|
||||
value = ""
|
||||
printf("%s=%s\n", key, value)
|
||||
printf("%s=%s\n",
|
||||
nilmdb.utils.unicode.encode(key),
|
||||
nilmdb.utils.unicode.encode(value))
|
||||
|
|
|
|||
|
|
@ -1,18 +1,15 @@
|
|||
import fnmatch
|
||||
|
||||
from nilmdb.utils.printf import printf
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.client
|
||||
|
||||
import fnmatch
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("remove", help="Remove data",
|
||||
description="""
|
||||
Remove all data from a specified time range within a
|
||||
stream. If multiple streams or wildcards are
|
||||
provided, the same time range is removed from all
|
||||
streams.
|
||||
stream. If multiple streams or wildcards are provided,
|
||||
the same time range is removed from all streams.
|
||||
""")
|
||||
cmd.set_defaults(handler=cmd_remove)
|
||||
cmd.set_defaults(handler = cmd_remove)
|
||||
|
||||
group = cmd.add_argument_group("Data selection")
|
||||
group.add_argument("path", nargs='+',
|
||||
|
|
@ -35,9 +32,8 @@ def setup(self, sub):
|
|||
help="Output number of data points removed")
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_remove(self):
|
||||
streams = [s[0] for s in self.client.stream_list()]
|
||||
streams = [ s[0] for s in self.client.stream_list() ]
|
||||
paths = []
|
||||
for path in self.args.path:
|
||||
new = fnmatch.filter(streams, path)
|
||||
|
|
@ -52,7 +48,7 @@ def cmd_remove(self):
|
|||
count = self.client.stream_remove(path,
|
||||
self.args.start, self.args.end)
|
||||
if self.args.count:
|
||||
printf("%d\n", count)
|
||||
printf("%d\n", count);
|
||||
except nilmdb.client.ClientError as e:
|
||||
self.die("error removing data: %s", str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.client
|
||||
|
||||
from argparse import ArgumentDefaultsHelpFormatter as def_form
|
||||
|
||||
def setup(self, sub):
|
||||
cmd = sub.add_parser("rename", help="Rename a stream",
|
||||
formatter_class=def_form,
|
||||
formatter_class = def_form,
|
||||
description="""
|
||||
Rename a stream.
|
||||
|
||||
Only the stream's path is renamed; no
|
||||
metadata is changed.
|
||||
""")
|
||||
cmd.set_defaults(handler=cmd_rename)
|
||||
cmd.set_defaults(handler = cmd_rename)
|
||||
group = cmd.add_argument_group("Required arguments")
|
||||
group.add_argument("oldpath",
|
||||
help="Old path, e.g. /foo/old",
|
||||
|
|
@ -23,7 +23,6 @@ def setup(self, sub):
|
|||
|
||||
return cmd
|
||||
|
||||
|
||||
def cmd_rename(self):
|
||||
"""Rename a stream"""
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
"""nilmdb.fsck"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from nilmdb.fsck.fsck import Fsck
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ import nilmdb.server
|
|||
import nilmdb.client.numpyclient
|
||||
from nilmdb.utils.interval import IntervalError
|
||||
from nilmdb.server.interval import Interval, IntervalSet
|
||||
from nilmdb.utils.printf import printf, fprintf, sprintf
|
||||
from nilmdb.utils.printf import *
|
||||
from nilmdb.utils.time import timestamp_to_string
|
||||
|
||||
from collections import defaultdict
|
||||
import sqlite3
|
||||
|
|
@ -18,83 +19,70 @@ import os
|
|||
import sys
|
||||
import progressbar
|
||||
import re
|
||||
import time
|
||||
import shutil
|
||||
import pickle
|
||||
import cPickle as pickle
|
||||
import numpy
|
||||
|
||||
|
||||
class FsckError(Exception):
|
||||
def __init__(self, msg="", *args):
|
||||
def __init__(self, msg = "", *args):
|
||||
if args:
|
||||
msg = sprintf(msg, *args)
|
||||
Exception.__init__(self, msg)
|
||||
|
||||
|
||||
class FixableFsckError(FsckError):
|
||||
def __init__(self, msg=""):
|
||||
FsckError.__init__(self, f'{msg}\nThis may be fixable with "--fix".')
|
||||
|
||||
|
||||
def __init__(self, msg = "", *args):
|
||||
if args:
|
||||
msg = sprintf(msg, *args)
|
||||
FsckError.__init__(self, "%s\nThis may be fixable with \"--fix\".", msg)
|
||||
class RetryFsck(FsckError):
|
||||
pass
|
||||
|
||||
|
||||
class FsckFormatError(FsckError):
|
||||
pass
|
||||
|
||||
|
||||
def log(format, *args):
|
||||
printf(format, *args)
|
||||
|
||||
|
||||
def err(format, *args):
|
||||
fprintf(sys.stderr, format, *args)
|
||||
|
||||
|
||||
# Decorator that retries a function if it returns a specific value
|
||||
def retry_if_raised(exc, message=None, max_retries=1000):
|
||||
def retry_if_raised(exc, message = None, max_retries = 100):
|
||||
def f1(func):
|
||||
def f2(*args, **kwargs):
|
||||
for n in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except exc:
|
||||
except exc as e:
|
||||
if message:
|
||||
log(f"{message} ({n+1})\n\n")
|
||||
raise Exception("Max number of retries (%d) exceeded; giving up" %
|
||||
max_retries)
|
||||
log("%s\n\n", message)
|
||||
raise Exception("Max number of retries (%d) exceeded; giving up")
|
||||
return f2
|
||||
return f1
|
||||
|
||||
|
||||
class Progress(object):
|
||||
def __init__(self, maxval):
|
||||
if maxval == 0:
|
||||
maxval = 1
|
||||
self.bar = progressbar.ProgressBar(
|
||||
maxval=maxval,
|
||||
widgets=[progressbar.Percentage(), ' ',
|
||||
maxval = maxval,
|
||||
widgets = [ progressbar.Percentage(), ' ',
|
||||
progressbar.Bar(), ' ',
|
||||
progressbar.ETA()])
|
||||
self.bar.term_width = self.bar.term_width or 75
|
||||
|
||||
progressbar.ETA() ])
|
||||
if self.bar.term_width == 0:
|
||||
self.bar.term_width = 75
|
||||
def __enter__(self):
|
||||
self.bar.start()
|
||||
self.last_update = 0
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
if exc_type is None:
|
||||
self.bar.finish()
|
||||
else:
|
||||
printf("\n")
|
||||
|
||||
def update(self, val):
|
||||
self.bar.update(val)
|
||||
|
||||
|
||||
class Fsck(object):
|
||||
def __init__(self, path, fix=False):
|
||||
|
||||
def __init__(self, path, fix = False):
|
||||
self.basepath = path
|
||||
self.sqlpath = os.path.join(path, "data.sql")
|
||||
self.bulkpath = os.path.join(path, "data")
|
||||
|
|
@ -104,7 +92,7 @@ class Fsck(object):
|
|||
### Main checks
|
||||
|
||||
@retry_if_raised(RetryFsck, "Something was fixed: restarting fsck")
|
||||
def check(self, skip_data=False):
|
||||
def check(self, skip_data = False):
|
||||
self.bulk = None
|
||||
self.sql = None
|
||||
try:
|
||||
|
|
@ -119,9 +107,7 @@ class Fsck(object):
|
|||
finally:
|
||||
if self.bulk:
|
||||
self.bulk.close()
|
||||
if self.sql: # pragma: no cover
|
||||
# (coverage doesn't handle finally clauses correctly;
|
||||
# both branches here are tested)
|
||||
if self.sql:
|
||||
self.sql.commit()
|
||||
self.sql.close()
|
||||
log("ok\n")
|
||||
|
|
@ -176,7 +162,7 @@ class Fsck(object):
|
|||
"ORDER BY start_time")
|
||||
for r in result:
|
||||
if r[0] not in self.stream_path:
|
||||
raise FsckError("interval ID %d not in streams", r[0])
|
||||
raise FsckError("interval ID %d not in streams", k)
|
||||
self.stream_interval[r[0]].append((r[1], r[2], r[3], r[4]))
|
||||
|
||||
log(" loading metadata\n")
|
||||
|
|
@ -184,17 +170,16 @@ class Fsck(object):
|
|||
result = cur.execute("SELECT stream_id, key, value FROM metadata")
|
||||
for r in result:
|
||||
if r[0] not in self.stream_path:
|
||||
raise FsckError("metadata ID %d not in streams", r[0])
|
||||
raise FsckError("metadata ID %d not in streams", k)
|
||||
if r[1] in self.stream_meta[r[0]]:
|
||||
raise FsckError(
|
||||
"duplicate metadata key '%s' for stream %d",
|
||||
raise FsckError("duplicate metadata key '%s' for stream %d",
|
||||
r[1], r[0])
|
||||
self.stream_meta[r[0]][r[1]] = r[2]
|
||||
|
||||
### Check streams and basic interval overlap
|
||||
|
||||
def check_streams(self):
|
||||
ids = list(self.stream_path.keys())
|
||||
ids = self.stream_path.keys()
|
||||
log("checking %s streams\n", "{:,d}".format(len(ids)))
|
||||
with Progress(len(ids)) as pbar:
|
||||
for i, sid in enumerate(ids):
|
||||
|
|
@ -202,7 +187,7 @@ class Fsck(object):
|
|||
path = self.stream_path[sid]
|
||||
|
||||
# unique path, valid layout
|
||||
if list(self.stream_path.values()).count(path) != 1:
|
||||
if self.stream_path.values().count(path) != 1:
|
||||
raise FsckError("duplicated path %s", path)
|
||||
layout = self.stream_layout[sid].split('_')[0]
|
||||
if layout not in ('int8', 'int16', 'int32', 'int64',
|
||||
|
|
@ -215,7 +200,6 @@ class Fsck(object):
|
|||
|
||||
# must exist in bulkdata
|
||||
bulk = self.bulkpath + path
|
||||
bulk = bulk.encode('utf-8')
|
||||
if not os.path.isdir(bulk):
|
||||
raise FsckError("%s: missing bulkdata dir", path)
|
||||
if not nilmdb.server.bulkdata.Table.exists(bulk):
|
||||
|
|
@ -238,97 +222,39 @@ class Fsck(object):
|
|||
try:
|
||||
posiset += new
|
||||
except IntervalError:
|
||||
self.fix_row_overlap(sid, path, posiset, new)
|
||||
raise FsckError("%s: overlap in file offsets:\n"
|
||||
"set: %s\nnew: %s",
|
||||
path, str(posiset), str(new))
|
||||
|
||||
try:
|
||||
# Check bulkdata
|
||||
# check bulkdata
|
||||
self.check_bulkdata(sid, path, bulk)
|
||||
|
||||
# Check that we can open bulkdata
|
||||
try:
|
||||
tab = None
|
||||
try:
|
||||
tab = nilmdb.server.bulkdata.Table(bulk)
|
||||
except FsckFormatError:
|
||||
# If there are no files except _format, try deleting
|
||||
# the entire stream; this may remove metadata, but
|
||||
# it's probably unimportant.
|
||||
files = list(os.listdir(bulk))
|
||||
if len(files) > 1:
|
||||
raise FsckFormatError(f"{path}: can't load _format, "
|
||||
f"but data is also present")
|
||||
|
||||
# Since the stream was empty, just remove it
|
||||
self.fix_remove_stream(sid, path, bulk,
|
||||
"empty, with corrupted format file")
|
||||
except FsckError as e:
|
||||
raise e
|
||||
except Exception as e: # pragma: no cover
|
||||
# No coverage because this is an unknown/unexpected error
|
||||
except Exception as e:
|
||||
raise FsckError("%s: can't open bulkdata: %s",
|
||||
path, str(e))
|
||||
finally:
|
||||
if tab:
|
||||
tab.close()
|
||||
|
||||
def fix_row_overlap(self, sid, path, existing, new):
|
||||
# If the file rows (spos, epos) overlap in the interval table,
|
||||
# and the overlapping ranges look like this:
|
||||
# A --------- C
|
||||
# B -------- D
|
||||
# Then we can try changing the first interval to go from
|
||||
# A to B instead.
|
||||
msg = (f"{path}: overlap in file offsets:\n"
|
||||
f"existing ranges: {existing}\n"
|
||||
f"overlapping interval: {new}")
|
||||
if not self.fix:
|
||||
raise FixableFsckError(msg)
|
||||
err(f"\n{msg}\nSeeing if we can truncate one of them...\n")
|
||||
|
||||
# See if there'e exactly one interval that overlaps the
|
||||
# conflicting one in the right way
|
||||
match = None
|
||||
for intv in self.stream_interval[sid]:
|
||||
(stime, etime, spos, epos) = intv
|
||||
if spos < new.start and epos > new.start:
|
||||
if match:
|
||||
err(f"no, more than one interval matched:\n"
|
||||
f"{intv}\n{match}\n")
|
||||
raise FsckError(f"{path}: unfixable overlap")
|
||||
match = intv
|
||||
if match is None:
|
||||
err("no intervals overlapped in the right way\n")
|
||||
raise FsckError(f"{path}: unfixable overlap")
|
||||
|
||||
# Truncate the file position
|
||||
err(f"truncating {match}\n")
|
||||
with self.sql:
|
||||
cur = self.sql.cursor()
|
||||
cur.execute("UPDATE ranges SET end_pos=? "
|
||||
"WHERE stream_id=? AND start_time=? AND "
|
||||
"end_time=? AND start_pos=? AND end_pos=?",
|
||||
(new.start, sid, *match))
|
||||
if cur.rowcount != 1: # pragma: no cover (shouldn't fail)
|
||||
raise FsckError("failed to fix SQL database")
|
||||
raise RetryFsck
|
||||
|
||||
### Check that bulkdata is good enough to be opened
|
||||
|
||||
@retry_if_raised(RetryFsck)
|
||||
def check_bulkdata(self, sid, path, bulk):
|
||||
try:
|
||||
with open(os.path.join(bulk, b"_format"), "rb") as f:
|
||||
with open(os.path.join(bulk, "_format"), "rb") as f:
|
||||
fmt = pickle.load(f)
|
||||
except Exception as e:
|
||||
raise FsckFormatError(f"{path}: can't load _format file ({e})")
|
||||
|
||||
if fmt["version"] != 3:
|
||||
raise FsckFormatError("%s: bad or unsupported bulkdata version %d",
|
||||
raise FsckError("%s: bad or unsupported bulkdata version %d",
|
||||
path, fmt["version"])
|
||||
rows_per_file = int(fmt["rows_per_file"])
|
||||
if rows_per_file < 1:
|
||||
raise FsckFormatError(f"{path}: bad rows_per_file {rows_per_file}")
|
||||
row_per_file = int(fmt["rows_per_file"])
|
||||
files_per_dir = int(fmt["files_per_dir"])
|
||||
if files_per_dir < 1:
|
||||
raise FsckFormatError(f"{path}: bad files_per_dir {files_per_dir}")
|
||||
layout = fmt["layout"]
|
||||
if layout != self.stream_layout[sid]:
|
||||
raise FsckFormatError("%s: layout mismatch %s != %s", path,
|
||||
raise FsckError("%s: layout mismatch %s != %s", path,
|
||||
layout, self.stream_layout[sid])
|
||||
|
||||
# Every file should have a size that's the multiple of the row size
|
||||
|
|
@ -337,16 +263,16 @@ class Fsck(object):
|
|||
rkt.close()
|
||||
|
||||
# Find all directories
|
||||
regex = re.compile(b"^[0-9a-f]{4,}$")
|
||||
regex = re.compile("^[0-9a-f]{4,}$")
|
||||
subdirs = sorted(filter(regex.search, os.listdir(bulk)),
|
||||
key=lambda x: int(x, 16), reverse=True)
|
||||
key = lambda x: int(x, 16), reverse = True)
|
||||
for subdir in subdirs:
|
||||
# Find all files in that dir
|
||||
subpath = os.path.join(bulk, subdir)
|
||||
files = list(filter(regex.search, os.listdir(subpath)))
|
||||
files = filter(regex.search, os.listdir(subpath))
|
||||
if not files:
|
||||
self.fix_empty_subdir(subpath)
|
||||
|
||||
raise RetryFsck
|
||||
# Verify that their size is a multiple of the row size
|
||||
for filename in files:
|
||||
filepath = os.path.join(subpath, filename)
|
||||
|
|
@ -362,11 +288,10 @@ class Fsck(object):
|
|||
# as long as it's only ".removed" files.
|
||||
err("\n%s\n", msg)
|
||||
for fn in os.listdir(subpath):
|
||||
if not fn.endswith(b".removed"):
|
||||
if not fn.endswith(".removed"):
|
||||
raise FsckError("can't fix automatically: please manually "
|
||||
"remove the file '%s' and try again",
|
||||
os.path.join(subpath, fn).decode(
|
||||
'utf-8', errors='backslashreplace'))
|
||||
"remove the file %s and try again",
|
||||
os.path.join(subpath, fn))
|
||||
# Remove the whole thing
|
||||
err("Removing empty subpath\n")
|
||||
shutil.rmtree(subpath)
|
||||
|
|
@ -387,40 +312,19 @@ class Fsck(object):
|
|||
f.truncate(newsize)
|
||||
raise RetryFsck
|
||||
|
||||
def fix_remove_stream(self, sid, path, bulk, reason):
|
||||
msg = f"stream {path} is corrupted: {reason}"
|
||||
if not self.fix:
|
||||
raise FixableFsckError(msg)
|
||||
# Remove the stream from disk and the database
|
||||
err(f"\n{msg}\n")
|
||||
err(f"Removing stream {path} from disk and database\n")
|
||||
shutil.rmtree(bulk)
|
||||
with self.sql:
|
||||
cur = self.sql.cursor()
|
||||
cur.execute("DELETE FROM streams WHERE id=?",
|
||||
(sid,))
|
||||
if cur.rowcount != 1: # pragma: no cover (shouldn't fail)
|
||||
raise FsckError("failed to remove stream")
|
||||
cur.execute("DELETE FROM ranges WHERE stream_id=?", (sid,))
|
||||
cur.execute("DELETE FROM metadata WHERE stream_id=?", (sid,))
|
||||
raise RetryFsck
|
||||
|
||||
### Check interval endpoints
|
||||
|
||||
def check_intervals(self):
|
||||
total_ints = sum(len(x) for x in list(self.stream_interval.values()))
|
||||
total_ints = sum(len(x) for x in self.stream_interval.values())
|
||||
log("checking %s intervals\n", "{:,d}".format(total_ints))
|
||||
done = 0
|
||||
with Progress(total_ints) as pbar:
|
||||
for sid in self.stream_interval:
|
||||
try:
|
||||
bulk = self.bulkpath + self.stream_path[sid]
|
||||
bulk = bulk.encode('utf-8')
|
||||
tab = nilmdb.server.bulkdata.Table(bulk)
|
||||
|
||||
def update(x):
|
||||
pbar.update(done + x)
|
||||
|
||||
ints = self.stream_interval[sid]
|
||||
done += self.check_table_intervals(sid, ints, tab, update)
|
||||
finally:
|
||||
|
|
@ -429,7 +333,7 @@ class Fsck(object):
|
|||
def check_table_intervals(self, sid, ints, tab, update):
|
||||
# look in the table to make sure we can pick out the interval's
|
||||
# endpoints
|
||||
path = self.stream_path[sid] # noqa: F841 unused
|
||||
path = self.stream_path[sid]
|
||||
tab.file_open.cache_remove_all()
|
||||
for (i, intv) in enumerate(ints):
|
||||
update(i)
|
||||
|
|
@ -437,11 +341,11 @@ class Fsck(object):
|
|||
if spos == epos and spos >= 0 and spos <= tab.nrows:
|
||||
continue
|
||||
try:
|
||||
srow = tab[spos] # noqa: F841 unused
|
||||
erow = tab[epos-1] # noqa: F841 unused
|
||||
srow = tab[spos]
|
||||
erow = tab[epos-1]
|
||||
except Exception as e:
|
||||
self.fix_bad_interval(sid, intv, tab, str(e))
|
||||
|
||||
raise RetryFsck
|
||||
return len(ints)
|
||||
|
||||
def fix_bad_interval(self, sid, intv, tab, msg):
|
||||
|
|
@ -470,23 +374,23 @@ class Fsck(object):
|
|||
"end_time=? AND start_pos=? AND end_pos=?",
|
||||
(new_etime, new_epos, sid, stime, etime,
|
||||
spos, epos))
|
||||
if cur.rowcount != 1: # pragma: no cover (shouldn't fail)
|
||||
if cur.rowcount != 1:
|
||||
raise FsckError("failed to fix SQL database")
|
||||
raise RetryFsck
|
||||
err("actually it can't be truncated; times are bad too\n")
|
||||
err("actually it can't be truncated; times are bad too")
|
||||
|
||||
# Otherwise, the only hope is to delete the interval entirely.
|
||||
err("*** Deleting the entire interval from SQL.\n")
|
||||
err("This may leave stale data on disk. To fix that, copy all "
|
||||
"data from this stream to a new stream using nilm-copy, then\n")
|
||||
err("remove all data from and destroy %s.\n", path)
|
||||
err("This may leave stale data on disk. To fix that, copy all\n")
|
||||
err("data from this stream to a new stream, then remove all data\n")
|
||||
err("from and destroy %s.\n", path)
|
||||
with self.sql:
|
||||
cur = self.sql.cursor()
|
||||
cur.execute("DELETE FROM ranges WHERE "
|
||||
"stream_id=? AND start_time=? AND "
|
||||
"end_time=? AND start_pos=? AND end_pos=?",
|
||||
(sid, stime, etime, spos, epos))
|
||||
if cur.rowcount != 1: # pragma: no cover (shouldn't fail)
|
||||
if cur.rowcount != 1:
|
||||
raise FsckError("failed to remove interval")
|
||||
raise RetryFsck
|
||||
|
||||
|
|
@ -494,19 +398,16 @@ class Fsck(object):
|
|||
|
||||
def check_data(self):
|
||||
total_rows = sum(sum((y[3] - y[2]) for y in x)
|
||||
for x in list(self.stream_interval.values()))
|
||||
for x in self.stream_interval.values())
|
||||
log("checking %s rows of data\n", "{:,d}".format(total_rows))
|
||||
done = 0
|
||||
with Progress(total_rows) as pbar:
|
||||
for sid in self.stream_interval:
|
||||
try:
|
||||
bulk = self.bulkpath + self.stream_path[sid]
|
||||
bulk = bulk.encode('utf-8')
|
||||
tab = nilmdb.server.bulkdata.Table(bulk)
|
||||
|
||||
def update(x):
|
||||
pbar.update(done + x)
|
||||
|
||||
ints = self.stream_interval[sid]
|
||||
done += self.check_table_data(sid, ints, tab, update)
|
||||
finally:
|
||||
|
|
@ -515,7 +416,7 @@ class Fsck(object):
|
|||
def check_table_data(self, sid, ints, tab, update):
|
||||
# Pull out all of the interval's data and verify that it's
|
||||
# monotonic.
|
||||
maxrows = getattr(self, 'maxrows_override', 100000)
|
||||
maxrows = 100000
|
||||
path = self.stream_path[sid]
|
||||
layout = self.stream_layout[sid]
|
||||
dtype = nilmdb.client.numpyclient.layout_to_dtype(layout)
|
||||
|
|
@ -535,76 +436,29 @@ class Fsck(object):
|
|||
|
||||
# Get raw data, convert to NumPy arary
|
||||
try:
|
||||
raw = tab.get_data(start, stop, binary=True)
|
||||
data = numpy.frombuffer(raw, dtype)
|
||||
except Exception as e: # pragma: no cover
|
||||
# No coverage because it's hard to trigger this -- earlier
|
||||
# checks check the ranges, so this would probably be a real
|
||||
# disk error, malloc failure, etc.
|
||||
raise FsckError(
|
||||
"%s: failed to grab rows %d through %d: %s",
|
||||
raw = tab.get_data(start, stop, binary = True)
|
||||
data = numpy.fromstring(raw, dtype)
|
||||
except Exception as e:
|
||||
raise FsckError("%s: failed to grab rows %d through %d: %s",
|
||||
path, start, stop, repr(e))
|
||||
|
||||
ts = data['timestamp']
|
||||
|
||||
# Verify that all timestamps are in range.
|
||||
match = (ts < stime) | (ts >= etime)
|
||||
if match.any():
|
||||
row = numpy.argmax(match)
|
||||
if ts[row] != 0:
|
||||
raise FsckError("%s: data timestamp %d at row %d "
|
||||
"outside interval range [%d,%d)",
|
||||
path, ts[row], row + start,
|
||||
stime, etime)
|
||||
|
||||
# Timestamp is zero and out of the expected range;
|
||||
# assume file ends with zeroed data and just truncate it.
|
||||
self.fix_table_by_truncating(
|
||||
path, tab, row + start,
|
||||
"data timestamp is out of range, and zero")
|
||||
|
||||
# Verify that timestamps are monotonic
|
||||
match = numpy.diff(ts) <= 0
|
||||
if match.any():
|
||||
row = numpy.argmax(match)
|
||||
if ts[row+1] != 0:
|
||||
raise FsckError(
|
||||
"%s: non-monotonic timestamp (%d -> %d) "
|
||||
"at row %d", path, ts[row], ts[row+1],
|
||||
row + start)
|
||||
|
||||
# Timestamp is zero and non-monotonic;
|
||||
# assume file ends with zeroed data and just truncate it.
|
||||
self.fix_table_by_truncating(
|
||||
path, tab, row + start + 1,
|
||||
"data timestamp is non-monotonic, and zero")
|
||||
|
||||
first_ts = ts[0]
|
||||
if (numpy.diff(data['timestamp']) <= 0).any():
|
||||
raise FsckError("%s: non-monotonic timestamp(s) in rows "
|
||||
"%d through %d", path, start, stop)
|
||||
first_ts = data['timestamp'][0]
|
||||
if last_ts is not None and first_ts <= last_ts:
|
||||
raise FsckError("%s: first interval timestamp %d is not "
|
||||
"greater than the previous last interval "
|
||||
"timestamp %d, at row %d",
|
||||
path, first_ts, last_ts, start)
|
||||
last_ts = ts[-1]
|
||||
last_ts = data['timestamp'][-1]
|
||||
|
||||
# The previous errors are fixable, by removing the
|
||||
# offending intervals, or changing the data
|
||||
# timestamps. But these are probably unlikely errors,
|
||||
# so it's not worth implementing that yet.
|
||||
# These are probably fixable, by removing the offending
|
||||
# intervals. But I'm not going to bother implementing
|
||||
# that yet.
|
||||
|
||||
# Done
|
||||
done += count
|
||||
update(done)
|
||||
return done
|
||||
|
||||
def fix_table_by_truncating(self, path, tab, row, reason):
|
||||
# Simple fix for bad data: truncate the table at the given row.
|
||||
# On retry, fix_bad_interval will correct the database and timestamps
|
||||
# to account for this truncation.
|
||||
msg = f"{path}: bad data in table, starting at row {row}: {reason}"
|
||||
if not self.fix:
|
||||
raise FixableFsckError(msg)
|
||||
err(f"\n{msg}\nWill try truncating table\n")
|
||||
(subdir, fname, offs, count) = tab._offset_from_row(row)
|
||||
tab._remove_or_truncate_file(subdir, fname, offs)
|
||||
raise RetryFsck
|
||||
|
|
|
|||
|
|
@ -1,27 +1,26 @@
|
|||
#!/usr/bin/env python3
|
||||
#!/usr/bin/python
|
||||
|
||||
import nilmdb.fsck
|
||||
import argparse
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
def main():
|
||||
"""Main entry point for the 'nilmdb-fsck' command line script"""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Check database consistency',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument("-v", "--version", action="version",
|
||||
version=nilmdb.__version__)
|
||||
description = 'Check database consistency',
|
||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
|
||||
version = nilmdb.__version__)
|
||||
parser.add_argument("-f", "--fix", action="store_true",
|
||||
default=False, help='Fix errors when possible '
|
||||
default=False, help = 'Fix errors when possible '
|
||||
'(which may involve removing data)')
|
||||
parser.add_argument("-n", "--no-data", action="store_true",
|
||||
default=False, help='Skip the slow full-data check')
|
||||
parser.add_argument('database', help='Database directory')
|
||||
default=False, help = 'Skip the slow full-data check')
|
||||
parser.add_argument('database', help = 'Database directory')
|
||||
args = parser.parse_args()
|
||||
|
||||
nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data=args.no_data)
|
||||
|
||||
nilmdb.fsck.Fsck(args.database, args.fix).check(skip_data = args.no_data)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,43 +1,36 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import socket
|
||||
import argparse
|
||||
|
||||
import cherrypy
|
||||
#!/usr/bin/python
|
||||
|
||||
import nilmdb.server
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import socket
|
||||
|
||||
def main():
|
||||
"""Main entry point for the 'nilmdb-server' command line script"""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Run the NilmDB server',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
|
||||
parser.add_argument("-v", "--version", action="version",
|
||||
version=nilmdb.__version__)
|
||||
description = 'Run the NilmDB server',
|
||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
|
||||
version = nilmdb.__version__)
|
||||
|
||||
group = parser.add_argument_group("Standard options")
|
||||
group.add_argument('-a', '--address',
|
||||
help='Only listen on the given address',
|
||||
default='0.0.0.0')
|
||||
group.add_argument('-p', '--port', help='Listen on the given port',
|
||||
type=int, default=12380)
|
||||
group.add_argument('-d', '--database', help='Database directory',
|
||||
default="./db")
|
||||
group.add_argument('-q', '--quiet', help='Silence output',
|
||||
action='store_true')
|
||||
help = 'Only listen on the given address',
|
||||
default = '0.0.0.0')
|
||||
group.add_argument('-p', '--port', help = 'Listen on the given port',
|
||||
type = int, default = 12380)
|
||||
group.add_argument('-d', '--database', help = 'Database directory',
|
||||
default = "./db")
|
||||
group.add_argument('-q', '--quiet', help = 'Silence output',
|
||||
action = 'store_true')
|
||||
group.add_argument('-t', '--traceback',
|
||||
help='Provide tracebacks in client errors',
|
||||
action='store_true', default=False)
|
||||
help = 'Provide tracebacks in client errors',
|
||||
action = 'store_true', default = False)
|
||||
|
||||
group = parser.add_argument_group("Debug options")
|
||||
group.add_argument('-y', '--yappi', help='Run under yappi profiler and '
|
||||
group.add_argument('-y', '--yappi', help = 'Run under yappi profiler and '
|
||||
'invoke interactive shell afterwards',
|
||||
action='store_true')
|
||||
action = 'store_true')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -46,54 +39,47 @@ def main():
|
|||
db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database)
|
||||
|
||||
# Configure the server
|
||||
if not args.quiet:
|
||||
cherrypy._cpconfig.environments['embedded']['log.screen'] = True
|
||||
|
||||
if args.quiet:
|
||||
embedded = True
|
||||
else:
|
||||
embedded = False
|
||||
server = nilmdb.server.Server(db,
|
||||
host=args.address,
|
||||
port=args.port,
|
||||
force_traceback=args.traceback)
|
||||
host = args.address,
|
||||
port = args.port,
|
||||
embedded = embedded,
|
||||
force_traceback = args.traceback)
|
||||
|
||||
# Print info
|
||||
if not args.quiet:
|
||||
print("Version: %s" % nilmdb.__version__)
|
||||
print("Database: %s" % (os.path.realpath(args.database)))
|
||||
print "Version: %s" % nilmdb.__version__
|
||||
print "Database: %s" % (os.path.realpath(args.database))
|
||||
if args.address == '0.0.0.0' or args.address == '::':
|
||||
host = socket.getfqdn()
|
||||
else:
|
||||
host = args.address
|
||||
print("Server URL: http://%s:%d/" % (host, args.port))
|
||||
print("----")
|
||||
print "Server URL: http://%s:%d/" % ( host, args.port)
|
||||
print "----"
|
||||
|
||||
# Run it
|
||||
try:
|
||||
if args.yappi:
|
||||
print("Running in yappi")
|
||||
print "Running in yappi"
|
||||
try:
|
||||
import yappi
|
||||
yappi.start()
|
||||
server.start(blocking=True)
|
||||
server.start(blocking = True)
|
||||
finally:
|
||||
yappi.stop()
|
||||
stats = yappi.get_func_stats()
|
||||
stats.sort("ttot")
|
||||
stats.print_all()
|
||||
try:
|
||||
yappi.print_stats(sort_type = yappi.SORTTYPE_TTOT, limit = 50)
|
||||
from IPython import embed
|
||||
embed(header="Use the `yappi` or `stats` object to "
|
||||
"explore further, `quit` to exit")
|
||||
except ModuleNotFoundError:
|
||||
print("\nInstall ipython to explore further")
|
||||
embed(header = "Use the yappi object to explore further, "
|
||||
"quit to exit")
|
||||
else:
|
||||
server.start(blocking=True)
|
||||
except nilmdb.server.serverutil.CherryPyExit:
|
||||
print("Exiting due to CherryPy error", file=sys.stderr)
|
||||
raise
|
||||
finally:
|
||||
if not args.quiet:
|
||||
print("Closing database")
|
||||
db.close()
|
||||
server.start(blocking = True)
|
||||
|
||||
# Clean up
|
||||
if not args.quiet:
|
||||
print "Closing database"
|
||||
db.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
#!/usr/bin/env python3
|
||||
#!/usr/bin/python
|
||||
|
||||
import nilmdb.cmdline
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the 'nilmtool' command line script"""
|
||||
nilmdb.cmdline.Cmdline().run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,8 +1,20 @@
|
|||
"""nilmdb.server"""
|
||||
|
||||
# Set up pyximport to automatically rebuild Cython modules if needed.
|
||||
import pyximport
|
||||
pyximport.install(inplace=True, build_in_temp=False)
|
||||
from __future__ import absolute_import
|
||||
|
||||
# Try to set up pyximport to automatically rebuild Cython modules. If
|
||||
# this doesn't work, it's OK, as long as the modules were built externally.
|
||||
# (e.g. python setup.py build_ext --inplace)
|
||||
try: # pragma: no cover
|
||||
import Cython
|
||||
import distutils.version
|
||||
if (distutils.version.LooseVersion(Cython.__version__) <
|
||||
distutils.version.LooseVersion("0.17")): # pragma: no cover
|
||||
raise ImportError("Cython version too old")
|
||||
import pyximport
|
||||
pyximport.install(inplace = True, build_in_temp = False)
|
||||
except (ImportError, TypeError): # pragma: no cover
|
||||
pass
|
||||
|
||||
from nilmdb.server.nilmdb import NilmDB
|
||||
from nilmdb.server.server import Server, wsgi_application
|
||||
|
|
|
|||
|
|
@ -1,15 +1,19 @@
|
|||
# Fixed record size bulk data storage
|
||||
|
||||
# Need absolute_import so that "import nilmdb" won't pull in
|
||||
# nilmdb.py, but will pull the parent nilmdb module instead.
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from nilmdb.utils.printf import *
|
||||
from nilmdb.utils.time import timestamp_to_string as timestamp_to_string
|
||||
import nilmdb.utils
|
||||
|
||||
import os
|
||||
import cPickle as pickle
|
||||
import re
|
||||
import sys
|
||||
import pickle
|
||||
import tempfile
|
||||
|
||||
from nilmdb.utils.printf import sprintf
|
||||
from nilmdb.utils.time import timestamp_to_string
|
||||
import nilmdb.utils
|
||||
|
||||
import nilmdb.utils.lock
|
||||
from . import rocket
|
||||
|
||||
|
|
@ -18,32 +22,28 @@ from . import rocket
|
|||
table_cache_size = 32
|
||||
fd_cache_size = 8
|
||||
|
||||
|
||||
@nilmdb.utils.must_close(wrap_verify=False)
|
||||
class BulkData():
|
||||
@nilmdb.utils.must_close(wrap_verify = False)
|
||||
class BulkData(object):
|
||||
def __init__(self, basepath, **kwargs):
|
||||
if isinstance(basepath, str):
|
||||
self.basepath = self._encode_filename(basepath)
|
||||
else:
|
||||
self.basepath = basepath
|
||||
self.root = os.path.join(self.basepath, b"data")
|
||||
self.lock = self.root + b".lock"
|
||||
self.root = os.path.join(self.basepath, "data")
|
||||
self.lock = self.root + ".lock"
|
||||
self.lockfile = None
|
||||
|
||||
# Tuneables
|
||||
if "file_size" in kwargs and kwargs["file_size"] is not None:
|
||||
if "file_size" in kwargs:
|
||||
self.file_size = kwargs["file_size"]
|
||||
else:
|
||||
# Default to approximately 128 MiB per file
|
||||
self.file_size = 128 * 1024 * 1024
|
||||
|
||||
if "files_per_dir" in kwargs and kwargs["files_per_dir"] is not None:
|
||||
if "files_per_dir" in kwargs:
|
||||
self.files_per_dir = kwargs["files_per_dir"]
|
||||
else:
|
||||
# 32768 files per dir should work even on FAT32
|
||||
self.files_per_dir = 32768
|
||||
|
||||
if "initial_nrows" in kwargs and kwargs["initial_nrows"] is not None:
|
||||
if "initial_nrows" in kwargs:
|
||||
self.initial_nrows = kwargs["initial_nrows"]
|
||||
else:
|
||||
# First row is 0
|
||||
|
|
@ -56,8 +56,7 @@ class BulkData():
|
|||
# Create the lock
|
||||
self.lockfile = open(self.lock, "w")
|
||||
if not nilmdb.utils.lock.exclusive_lock(self.lockfile):
|
||||
raise IOError('database at "' +
|
||||
self._decode_filename(self.basepath) +
|
||||
raise IOError('database at "' + self.basepath +
|
||||
'" is already locked by another process')
|
||||
|
||||
def close(self):
|
||||
|
|
@ -67,21 +66,21 @@ class BulkData():
|
|||
self.lockfile.close()
|
||||
try:
|
||||
os.unlink(self.lock)
|
||||
except OSError:
|
||||
except OSError: # pragma: no cover
|
||||
pass
|
||||
self.lockfile = None
|
||||
|
||||
def _encode_filename(self, path):
|
||||
# Translate unicode strings to raw bytes, if needed. We
|
||||
# always manipulate paths internally as bytes.
|
||||
# Encode all paths to UTF-8, regardless of sys.getfilesystemencoding(),
|
||||
# because we want to be able to represent all code points and the user
|
||||
# will never be directly exposed to filenames. We can then do path
|
||||
# manipulations on the UTF-8 directly.
|
||||
if isinstance(path, unicode):
|
||||
return path.encode('utf-8')
|
||||
|
||||
def _decode_filename(self, path):
|
||||
# Translate raw bytes to unicode strings, escaping if needed
|
||||
return path.decode('utf-8', errors='backslashreplace')
|
||||
return path
|
||||
|
||||
def _create_check_ospath(self, ospath):
|
||||
if ospath[-1:] == b'/':
|
||||
if ospath[-1] == '/':
|
||||
raise ValueError("invalid path; should not end with a /")
|
||||
if Table.exists(ospath):
|
||||
raise ValueError("stream already exists at this path")
|
||||
|
|
@ -89,7 +88,7 @@ class BulkData():
|
|||
# Look for any files in subdirectories. Fully empty subdirectories
|
||||
# are OK; they might be there during a rename
|
||||
for (root, dirs, files) in os.walk(ospath):
|
||||
if files:
|
||||
if len(files):
|
||||
raise ValueError(
|
||||
"non-empty subdirs of this path already exist")
|
||||
|
||||
|
|
@ -98,13 +97,13 @@ class BulkData():
|
|||
don't exist. Returns a list of elements that got created."""
|
||||
path = self._encode_filename(unicodepath)
|
||||
|
||||
if path[0:1] != b'/':
|
||||
raise ValueError("paths must start with / ")
|
||||
[group, node] = path.rsplit(b"/", 1)
|
||||
if group == b'':
|
||||
if path[0] != '/':
|
||||
raise ValueError("paths must start with /")
|
||||
[ group, node ] = path.rsplit("/", 1)
|
||||
if group == '':
|
||||
raise ValueError("invalid path; path must contain at least one "
|
||||
"folder")
|
||||
if node == b'':
|
||||
if node == '':
|
||||
raise ValueError("invalid path; should not end with a /")
|
||||
if not Table.valid_path(path):
|
||||
raise ValueError("path name is invalid or contains reserved words")
|
||||
|
|
@ -115,7 +114,7 @@ class BulkData():
|
|||
# os.path.join)
|
||||
|
||||
# Make directories leading up to this one
|
||||
elements = path.lstrip(b'/').split(b'/')
|
||||
elements = path.lstrip('/').split('/')
|
||||
made_dirs = []
|
||||
try:
|
||||
# Make parent elements
|
||||
|
|
@ -126,11 +125,15 @@ class BulkData():
|
|||
if not os.path.isdir(ospath):
|
||||
os.mkdir(ospath)
|
||||
made_dirs.append(ospath)
|
||||
except Exception:
|
||||
# Remove paths that we created
|
||||
for ospath in reversed(made_dirs):
|
||||
except Exception as e:
|
||||
# Try to remove paths that we created; ignore errors
|
||||
exc_info = sys.exc_info()
|
||||
for ospath in reversed(made_dirs): # pragma: no cover (hard to hit)
|
||||
try:
|
||||
os.rmdir(ospath)
|
||||
raise
|
||||
except OSError:
|
||||
pass
|
||||
raise exc_info[1], None, exc_info[2]
|
||||
|
||||
return elements
|
||||
|
||||
|
|
@ -165,7 +168,7 @@ class BulkData():
|
|||
os.rmdir(ospath)
|
||||
except OSError:
|
||||
pass
|
||||
raise exc_info[1].with_traceback(exc_info[2])
|
||||
raise exc_info[1], None, exc_info[2]
|
||||
|
||||
# Success
|
||||
return
|
||||
|
|
@ -173,8 +176,8 @@ class BulkData():
|
|||
def _remove_leaves(self, unicodepath):
|
||||
"""Remove empty directories starting at the leaves of unicodepath"""
|
||||
path = self._encode_filename(unicodepath)
|
||||
elements = path.lstrip(b'/').split(b'/')
|
||||
for i in reversed(list(range(len(elements)))):
|
||||
elements = path.lstrip('/').split('/')
|
||||
for i in reversed(range(len(elements))):
|
||||
ospath = os.path.join(self.root, *elements[0:i+1])
|
||||
try:
|
||||
os.rmdir(ospath)
|
||||
|
|
@ -188,9 +191,9 @@ class BulkData():
|
|||
newpath = self._encode_filename(newunicodepath)
|
||||
|
||||
# Get OS paths
|
||||
oldelements = oldpath.lstrip(b'/').split(b'/')
|
||||
oldelements = oldpath.lstrip('/').split('/')
|
||||
oldospath = os.path.join(self.root, *oldelements)
|
||||
newelements = newpath.lstrip(b'/').split(b'/')
|
||||
newelements = newpath.lstrip('/').split('/')
|
||||
newospath = os.path.join(self.root, *newelements)
|
||||
|
||||
# Basic checks
|
||||
|
|
@ -201,8 +204,8 @@ class BulkData():
|
|||
self.getnode.cache_remove(self, oldunicodepath)
|
||||
|
||||
# Move the table to a temporary location
|
||||
tmpdir = tempfile.mkdtemp(prefix=b"rename-", dir=self.root)
|
||||
tmppath = os.path.join(tmpdir, b"table")
|
||||
tmpdir = tempfile.mkdtemp(prefix = "rename-", dir = self.root)
|
||||
tmppath = os.path.join(tmpdir, "table")
|
||||
os.rename(oldospath, tmppath)
|
||||
|
||||
try:
|
||||
|
|
@ -230,7 +233,7 @@ class BulkData():
|
|||
path = self._encode_filename(unicodepath)
|
||||
|
||||
# Get OS path
|
||||
elements = path.lstrip(b'/').split(b'/')
|
||||
elements = path.lstrip('/').split('/')
|
||||
ospath = os.path.join(self.root, *elements)
|
||||
|
||||
# Remove Table object from cache
|
||||
|
|
@ -239,7 +242,7 @@ class BulkData():
|
|||
# Remove the contents of the target directory
|
||||
if not Table.exists(ospath):
|
||||
raise ValueError("nothing at that path")
|
||||
for (root, dirs, files) in os.walk(ospath, topdown=False):
|
||||
for (root, dirs, files) in os.walk(ospath, topdown = False):
|
||||
for name in files:
|
||||
os.remove(os.path.join(root, name))
|
||||
for name in dirs:
|
||||
|
|
@ -249,19 +252,18 @@ class BulkData():
|
|||
self._remove_leaves(unicodepath)
|
||||
|
||||
# Cache open tables
|
||||
@nilmdb.utils.lru_cache(size=table_cache_size,
|
||||
onremove=lambda x: x.close())
|
||||
@nilmdb.utils.lru_cache(size = table_cache_size,
|
||||
onremove = lambda x: x.close())
|
||||
def getnode(self, unicodepath):
|
||||
"""Return a Table object corresponding to the given database
|
||||
path, which must exist."""
|
||||
path = self._encode_filename(unicodepath)
|
||||
elements = path.lstrip(b'/').split(b'/')
|
||||
elements = path.lstrip('/').split('/')
|
||||
ospath = os.path.join(self.root, *elements)
|
||||
return Table(ospath, self.initial_nrows)
|
||||
|
||||
|
||||
@nilmdb.utils.must_close(wrap_verify=False)
|
||||
class Table():
|
||||
@nilmdb.utils.must_close(wrap_verify = False)
|
||||
class Table(object):
|
||||
"""Tools to help access a single table (data at a specific OS path)."""
|
||||
# See design.md for design details
|
||||
|
||||
|
|
@ -269,12 +271,12 @@ class Table():
|
|||
@classmethod
|
||||
def valid_path(cls, root):
|
||||
"""Return True if a root path is a valid name"""
|
||||
return b"_format" not in root.split(b"/")
|
||||
return "_format" not in root.split("/")
|
||||
|
||||
@classmethod
|
||||
def exists(cls, root):
|
||||
"""Return True if a table appears to exist at this OS path"""
|
||||
return os.path.isfile(os.path.join(root, b"_format"))
|
||||
return os.path.isfile(os.path.join(root, "_format"))
|
||||
|
||||
@classmethod
|
||||
def create(cls, root, layout, file_size, files_per_dir):
|
||||
|
|
@ -287,26 +289,24 @@ class Table():
|
|||
rows_per_file = max(file_size // rkt.binary_size, 1)
|
||||
rkt.close()
|
||||
|
||||
fmt = {
|
||||
"rows_per_file": rows_per_file,
|
||||
fmt = { "rows_per_file": rows_per_file,
|
||||
"files_per_dir": files_per_dir,
|
||||
"layout": layout,
|
||||
"version": 3
|
||||
}
|
||||
nilmdb.utils.atomic.replace_file(
|
||||
os.path.join(root, b"_format"), pickle.dumps(fmt, 2))
|
||||
"version": 3 }
|
||||
with open(os.path.join(root, "_format"), "wb") as f:
|
||||
pickle.dump(fmt, f, 2)
|
||||
|
||||
# Normal methods
|
||||
def __init__(self, root, initial_nrows=0):
|
||||
def __init__(self, root, initial_nrows = 0):
|
||||
"""'root' is the full OS path to the directory of this table"""
|
||||
self.root = root
|
||||
self.initial_nrows = initial_nrows
|
||||
|
||||
# Load the format
|
||||
with open(os.path.join(self.root, b"_format"), "rb") as f:
|
||||
with open(os.path.join(self.root, "_format"), "rb") as f:
|
||||
fmt = pickle.load(f)
|
||||
|
||||
if fmt["version"] != 3:
|
||||
if fmt["version"] != 3: # pragma: no cover
|
||||
# Old versions used floating point timestamps, which aren't
|
||||
# valid anymore.
|
||||
raise NotImplementedError("old version " + str(fmt["version"]) +
|
||||
|
|
@ -336,25 +336,25 @@ class Table():
|
|||
# greater than the row number of any piece of data that
|
||||
# currently exists, not necessarily all data that _ever_
|
||||
# existed.
|
||||
regex = re.compile(b"^[0-9a-f]{4,}$")
|
||||
regex = re.compile("^[0-9a-f]{4,}$")
|
||||
|
||||
# Find the last directory. We sort and loop through all of them,
|
||||
# starting with the numerically greatest, because the dirs could be
|
||||
# empty if something was deleted but the directory was unexpectedly
|
||||
# not deleted.
|
||||
subdirs = sorted(filter(regex.search, os.listdir(self.root)),
|
||||
key=lambda x: int(x, 16), reverse=True)
|
||||
key = lambda x: int(x, 16), reverse = True)
|
||||
|
||||
for subdir in subdirs:
|
||||
# Now find the last file in that dir
|
||||
path = os.path.join(self.root, subdir)
|
||||
files = list(filter(regex.search, os.listdir(path)))
|
||||
if not files:
|
||||
files = filter(regex.search, os.listdir(path))
|
||||
if not files: # pragma: no cover (shouldn't occur)
|
||||
# Empty dir: try the next one
|
||||
continue
|
||||
|
||||
# Find the numerical max
|
||||
filename = max(files, key=lambda x: int(x, 16))
|
||||
filename = max(files, key = lambda x: int(x, 16))
|
||||
offset = os.path.getsize(os.path.join(self.root, subdir, filename))
|
||||
|
||||
# Convert to row number
|
||||
|
|
@ -380,8 +380,8 @@ class Table():
|
|||
filenum = row // self.rows_per_file
|
||||
# It's OK if these format specifiers are too short; the filenames
|
||||
# will just get longer but will still sort correctly.
|
||||
dirname = sprintf(b"%04x", filenum // self.files_per_dir)
|
||||
filename = sprintf(b"%04x", filenum % self.files_per_dir)
|
||||
dirname = sprintf("%04x", filenum // self.files_per_dir)
|
||||
filename = sprintf("%04x", filenum % self.files_per_dir)
|
||||
offset = (row % self.rows_per_file) * self.row_size
|
||||
count = self.rows_per_file - (row % self.rows_per_file)
|
||||
return (dirname, filename, offset, count)
|
||||
|
|
@ -389,14 +389,14 @@ class Table():
|
|||
def _row_from_offset(self, subdir, filename, offset):
|
||||
"""Return the row number that corresponds to the given
|
||||
'subdir/filename' and byte-offset within that file."""
|
||||
if (offset % self.row_size) != 0:
|
||||
if (offset % self.row_size) != 0: # pragma: no cover
|
||||
# this shouldn't occur, unless there is some corruption somewhere
|
||||
raise ValueError("file offset is not a multiple of data size")
|
||||
filenum = int(subdir, 16) * self.files_per_dir + int(filename, 16)
|
||||
row = (filenum * self.rows_per_file) + (offset // self.row_size)
|
||||
return row
|
||||
|
||||
def _remove_or_truncate_file(self, subdir, filename, offset=0):
|
||||
def _remove_or_truncate_file(self, subdir, filename, offset = 0):
|
||||
"""Remove the given file, and remove the subdirectory too
|
||||
if it's empty. If offset is nonzero, truncate the file
|
||||
to that size instead."""
|
||||
|
|
@ -416,8 +416,8 @@ class Table():
|
|||
pass
|
||||
|
||||
# Cache open files
|
||||
@nilmdb.utils.lru_cache(size=fd_cache_size,
|
||||
onremove=lambda f: f.close())
|
||||
@nilmdb.utils.lru_cache(size = fd_cache_size,
|
||||
onremove = lambda f: f.close())
|
||||
def file_open(self, subdir, filename):
|
||||
"""Open and map a given 'subdir/filename' (relative to self.root).
|
||||
Will be automatically closed when evicted from the cache."""
|
||||
|
|
@ -430,14 +430,12 @@ class Table():
|
|||
return rocket.Rocket(self.layout,
|
||||
os.path.join(self.root, subdir, filename))
|
||||
|
||||
def append_data(self, data, start, end, binary=False):
|
||||
def append_data(self, data, start, end, binary = False):
|
||||
"""Parse the formatted string in 'data', according to the
|
||||
current layout, and append it to the table. If any timestamps
|
||||
are non-monotonic, or don't fall between 'start' and 'end',
|
||||
a ValueError is raised.
|
||||
|
||||
Note that data is always of 'bytes' type.
|
||||
|
||||
If 'binary' is True, the data should be in raw binary format
|
||||
instead: little-endian, matching the current table's layout,
|
||||
including the int64 timestamp.
|
||||
|
|
@ -454,7 +452,7 @@ class Table():
|
|||
while data_offset < len(data):
|
||||
# See how many rows we can fit into the current file,
|
||||
# and open it
|
||||
(subdir, fname, offs, count) = self._offset_from_row(tot_rows)
|
||||
(subdir, fname, offset, count) = self._offset_from_row(tot_rows)
|
||||
f = self.file_open(subdir, fname)
|
||||
|
||||
# Ask the rocket object to parse and append up to "count"
|
||||
|
|
@ -478,9 +476,9 @@ class Table():
|
|||
if binary:
|
||||
raise IndexError
|
||||
bad = data.splitlines()[linenum-1]
|
||||
bad += b'\n' + b' ' * (colnum - 1) + b'^'
|
||||
bad += '\n' + ' ' * (colnum - 1) + '^'
|
||||
except IndexError:
|
||||
bad = b""
|
||||
bad = ""
|
||||
if errtype == rocket.ERR_NON_MONOTONIC:
|
||||
err = "timestamp is not monotonically increasing"
|
||||
elif errtype == rocket.ERR_OUT_OF_INTERVAL:
|
||||
|
|
@ -494,17 +492,16 @@ class Table():
|
|||
timestamp_to_string(end))
|
||||
else:
|
||||
err = str(obj)
|
||||
bad_str = bad.decode('utf-8', errors='backslashreplace')
|
||||
raise ValueError("error parsing input data: " +
|
||||
where + err + "\n" + bad_str)
|
||||
where + err + "\n" + bad)
|
||||
tot_rows += added_rows
|
||||
except Exception:
|
||||
# Some failure, so try to roll things back by truncating or
|
||||
# deleting files that we may have appended data to.
|
||||
cleanpos = self.nrows
|
||||
while cleanpos <= tot_rows:
|
||||
(subdir, fname, offs, count) = self._offset_from_row(cleanpos)
|
||||
self._remove_or_truncate_file(subdir, fname, offs)
|
||||
(subdir, fname, offset, count) = self._offset_from_row(cleanpos)
|
||||
self._remove_or_truncate_file(subdir, fname, offset)
|
||||
cleanpos += count
|
||||
# Re-raise original exception
|
||||
raise
|
||||
|
|
@ -512,11 +509,14 @@ class Table():
|
|||
# Success, so update self.nrows accordingly
|
||||
self.nrows = tot_rows
|
||||
|
||||
def get_data(self, start, stop, binary=False):
|
||||
def get_data(self, start, stop, binary = False):
|
||||
"""Extract data corresponding to Python range [n:m],
|
||||
and returns a formatted string"""
|
||||
if (start is None or stop is None or
|
||||
start > stop or start < 0 or stop > self.nrows):
|
||||
if (start is None or
|
||||
stop is None or
|
||||
start > stop or
|
||||
start < 0 or
|
||||
stop > self.nrows):
|
||||
raise IndexError("Index out of range")
|
||||
|
||||
ret = []
|
||||
|
|
@ -556,7 +556,7 @@ class Table():
|
|||
# file. Only when the list covers the entire extent of the
|
||||
# file will that file be removed.
|
||||
datafile = os.path.join(self.root, subdir, filename)
|
||||
cachefile = datafile + b".removed"
|
||||
cachefile = datafile + ".removed"
|
||||
try:
|
||||
with open(cachefile, "rb") as f:
|
||||
ranges = pickle.load(f)
|
||||
|
|
@ -583,8 +583,7 @@ class Table():
|
|||
# Not connected; append previous and start again
|
||||
merged.append(prev)
|
||||
prev = new
|
||||
# Last range we were looking at goes into the file. We know
|
||||
# there was at least one (the one we just removed).
|
||||
if prev is not None:
|
||||
merged.append(prev)
|
||||
|
||||
# If the range covered the whole file, we can delete it now.
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
"""Exceptions"""
|
||||
|
||||
|
||||
class NilmDBError(Exception):
|
||||
"""Base exception for NilmDB errors"""
|
||||
def __init__(self, msg="Unspecified error"):
|
||||
super().__init__(msg)
|
||||
|
||||
def __init__(self, message = "Unspecified error"):
|
||||
Exception.__init__(self, message)
|
||||
|
||||
class StreamError(NilmDBError):
|
||||
pass
|
||||
|
||||
|
||||
class OverlapError(NilmDBError):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# cython: language_level=2
|
||||
|
||||
"""Interval, IntervalSet
|
||||
|
||||
The Interval implemented here is just like
|
||||
|
|
@ -60,19 +58,9 @@ cdef class Interval:
|
|||
return ("[" + timestamp_to_string(self.start) +
|
||||
" -> " + timestamp_to_string(self.end) + ")")
|
||||
|
||||
# Compare two intervals. If non-equal, order by start then end
|
||||
def __lt__(self, Interval other):
|
||||
return (self.start, self.end) < (other.start, other.end)
|
||||
def __gt__(self, Interval other):
|
||||
return (self.start, self.end) > (other.start, other.end)
|
||||
def __le__(self, Interval other):
|
||||
return (self.start, self.end) <= (other.start, other.end)
|
||||
def __ge__(self, Interval other):
|
||||
return (self.start, self.end) >= (other.start, other.end)
|
||||
def __eq__(self, Interval other):
|
||||
return (self.start, self.end) == (other.start, other.end)
|
||||
def __ne__(self, Interval other):
|
||||
return (self.start, self.end) != (other.start, other.end)
|
||||
def __cmp__(self, Interval other):
|
||||
"""Compare two intervals. If non-equal, order by start then end"""
|
||||
return cmp(self.start, other.start) or cmp(self.end, other.end)
|
||||
|
||||
cpdef intersects(self, Interval other):
|
||||
"""Return True if two Interval objects intersect"""
|
||||
|
|
|
|||
|
|
@ -7,13 +7,12 @@ Object that represents a NILM database file.
|
|||
Manages both the SQL database and the table storage backend.
|
||||
"""
|
||||
|
||||
import os
|
||||
import errno
|
||||
import sqlite3
|
||||
|
||||
# Need absolute_import so that "import nilmdb" won't pull in
|
||||
# nilmdb.py, but will pull the parent nilmdb module instead.
|
||||
from __future__ import absolute_import
|
||||
import nilmdb.utils
|
||||
from nilmdb.utils.printf import printf
|
||||
from nilmdb.utils.time import timestamp_to_bytes
|
||||
from nilmdb.utils.printf import *
|
||||
from nilmdb.utils.time import timestamp_to_string
|
||||
|
||||
from nilmdb.utils.interval import IntervalError
|
||||
from nilmdb.server.interval import Interval, DBInterval, IntervalSet
|
||||
|
|
@ -21,6 +20,10 @@ from nilmdb.server.interval import Interval, DBInterval, IntervalSet
|
|||
from nilmdb.server import bulkdata
|
||||
from nilmdb.server.errors import NilmDBError, StreamError, OverlapError
|
||||
|
||||
import sqlite3
|
||||
import os
|
||||
import errno
|
||||
|
||||
# Note about performance and transactions:
|
||||
#
|
||||
# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
|
||||
|
|
@ -34,7 +37,7 @@ from nilmdb.server.errors import NilmDBError, StreamError, OverlapError
|
|||
# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL'
|
||||
# give an equivalent speedup more safely. That is what is used here.
|
||||
_sql_schema_updates = {
|
||||
0: {"next": 1, "sql": """
|
||||
0: { "next": 1, "sql": """
|
||||
-- All streams
|
||||
CREATE TABLE streams(
|
||||
id INTEGER PRIMARY KEY, -- stream ID
|
||||
|
|
@ -58,26 +61,25 @@ _sql_schema_updates = {
|
|||
end_pos INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX _ranges_index ON ranges (stream_id, start_time, end_time);
|
||||
"""},
|
||||
""" },
|
||||
|
||||
1: {"next": 3, "sql": """
|
||||
1: { "next": 3, "sql": """
|
||||
-- Generic dictionary-type metadata that can be associated with a stream
|
||||
CREATE TABLE metadata(
|
||||
stream_id INTEGER NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
value TEXT
|
||||
);
|
||||
"""},
|
||||
""" },
|
||||
|
||||
2: {"error": "old format with floating-point timestamps requires "
|
||||
"nilmdb 1.3.1 or older"},
|
||||
2: { "error": "old format with floating-point timestamps requires "
|
||||
"nilmdb 1.3.1 or older" },
|
||||
|
||||
3: {"next": None},
|
||||
3: { "next": None },
|
||||
}
|
||||
|
||||
|
||||
@nilmdb.utils.must_close()
|
||||
class NilmDB():
|
||||
class NilmDB(object):
|
||||
verbose = 0
|
||||
|
||||
def __init__(self, basepath,
|
||||
|
|
@ -109,7 +111,9 @@ class NilmDB():
|
|||
try:
|
||||
os.makedirs(self.basepath)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
if e.errno != errno.EEXIST: # pragma: no cover
|
||||
# (no coverage, because it's hard to trigger this case
|
||||
# if tests are run as root)
|
||||
raise IOError("can't create tree " + self.basepath)
|
||||
|
||||
# Our data goes inside it
|
||||
|
|
@ -117,10 +121,10 @@ class NilmDB():
|
|||
|
||||
# SQLite database too
|
||||
sqlfilename = os.path.join(self.basepath, "data.sql")
|
||||
self.con = sqlite3.connect(sqlfilename, check_same_thread=True)
|
||||
self.con = sqlite3.connect(sqlfilename, check_same_thread = True)
|
||||
try:
|
||||
self._sql_schema_update()
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
self.data.close()
|
||||
raise
|
||||
|
||||
|
|
@ -145,7 +149,6 @@ class NilmDB():
|
|||
if self.con:
|
||||
self.con.commit()
|
||||
self.con.close()
|
||||
self.con = None
|
||||
self.data.close()
|
||||
|
||||
def _sql_schema_update(self):
|
||||
|
|
@ -154,18 +157,18 @@ class NilmDB():
|
|||
oldversion = version
|
||||
|
||||
while True:
|
||||
if version not in _sql_schema_updates:
|
||||
if version not in _sql_schema_updates: # pragma: no cover
|
||||
raise Exception(self.basepath + ": unknown database version "
|
||||
+ str(version))
|
||||
update = _sql_schema_updates[version]
|
||||
if "error" in update:
|
||||
if "error" in update: # pragma: no cover
|
||||
raise Exception(self.basepath + ": can't use database version "
|
||||
+ str(version) + ": " + update["error"])
|
||||
if update["next"] is None:
|
||||
break
|
||||
cur.executescript(update["sql"])
|
||||
version = update["next"]
|
||||
if self.verbose:
|
||||
if self.verbose: # pragma: no cover
|
||||
printf("Database schema updated to %d\n", version)
|
||||
|
||||
if version != oldversion:
|
||||
|
|
@ -181,7 +184,7 @@ class NilmDB():
|
|||
raise NilmDBError("start must precede end")
|
||||
return (start, end)
|
||||
|
||||
@nilmdb.utils.lru_cache(size=64)
|
||||
@nilmdb.utils.lru_cache(size = 64)
|
||||
def _get_intervals(self, stream_id):
|
||||
"""
|
||||
Return a mutable IntervalSet corresponding to the given stream ID.
|
||||
|
|
@ -196,7 +199,7 @@ class NilmDB():
|
|||
iset += DBInterval(start_time, end_time,
|
||||
start_time, end_time,
|
||||
start_pos, end_pos)
|
||||
except IntervalError:
|
||||
except IntervalError: # pragma: no cover
|
||||
raise NilmDBError("unexpected overlap in ranges table!")
|
||||
|
||||
return iset
|
||||
|
|
@ -223,6 +226,10 @@ class NilmDB():
|
|||
# Load this stream's intervals
|
||||
iset = self._get_intervals(stream_id)
|
||||
|
||||
# Check for overlap
|
||||
if iset.intersects(interval): # pragma: no cover (gets caught earlier)
|
||||
raise NilmDBError("new interval overlaps existing data")
|
||||
|
||||
# Check for adjacency. If there's a stream in the database
|
||||
# that ends exactly when this one starts, and the database
|
||||
# rows match up, we can make one interval that covers the
|
||||
|
|
@ -265,6 +272,10 @@ class NilmDB():
|
|||
original: original DBInterval; must be already present in DB
|
||||
to_remove: DBInterval to remove; must be subset of 'original'
|
||||
"""
|
||||
# Just return if we have nothing to remove
|
||||
if remove.start == remove.end: # pragma: no cover
|
||||
return
|
||||
|
||||
# Load this stream's intervals
|
||||
iset = self._get_intervals(stream_id)
|
||||
|
||||
|
|
@ -279,8 +290,7 @@ class NilmDB():
|
|||
# the removed piece was in the middle.
|
||||
def add(iset, start, end, start_pos, end_pos):
|
||||
iset += DBInterval(start, end, start, end, start_pos, end_pos)
|
||||
self._sql_interval_insert(stream_id, start, end,
|
||||
start_pos, end_pos)
|
||||
self._sql_interval_insert(stream_id, start, end, start_pos, end_pos)
|
||||
|
||||
if original.start != remove.start:
|
||||
# Interval before the removed region
|
||||
|
|
@ -297,7 +307,7 @@ class NilmDB():
|
|||
|
||||
return
|
||||
|
||||
def stream_list(self, path=None, layout=None, extended=False):
|
||||
def stream_list(self, path = None, layout = None, extended = False):
|
||||
"""Return list of lists of all streams in the database.
|
||||
|
||||
If path is specified, include only streams with a path that
|
||||
|
|
@ -306,10 +316,10 @@ class NilmDB():
|
|||
If layout is specified, include only streams with a layout
|
||||
that matches the given string.
|
||||
|
||||
If extended=False, returns a list of lists containing
|
||||
If extended = False, returns a list of lists containing
|
||||
the path and layout: [ path, layout ]
|
||||
|
||||
If extended=True, returns a list of lists containing
|
||||
If extended = True, returns a list of lists containing
|
||||
more information:
|
||||
path
|
||||
layout
|
||||
|
|
@ -336,9 +346,9 @@ class NilmDB():
|
|||
params += (path,)
|
||||
query += " GROUP BY streams.id ORDER BY streams.path"
|
||||
result = self.con.execute(query, params).fetchall()
|
||||
return [list(x) for x in result]
|
||||
return [ list(x) for x in result ]
|
||||
|
||||
def stream_intervals(self, path, start=None, end=None, diffpath=None):
|
||||
def stream_intervals(self, path, start = None, end = None, diffpath = None):
|
||||
"""
|
||||
List all intervals in 'path' between 'start' and 'end'. If
|
||||
'diffpath' is not none, list instead the set-difference
|
||||
|
|
@ -410,8 +420,8 @@ class NilmDB():
|
|||
|
||||
def stream_set_metadata(self, path, data):
|
||||
"""Set stream metadata from a dictionary, e.g.
|
||||
{ description: 'Downstairs lighting',
|
||||
v_scaling: 123.45 }
|
||||
{ description = 'Downstairs lighting',
|
||||
v_scaling = 123.45 }
|
||||
This replaces all existing metadata.
|
||||
"""
|
||||
stream_id = self._stream_id(path)
|
||||
|
|
@ -459,7 +469,7 @@ class NilmDB():
|
|||
|
||||
# Verify that no intervals are present, and clear the cache
|
||||
iset = self._get_intervals(stream_id)
|
||||
if iset:
|
||||
if len(iset):
|
||||
raise NilmDBError("all intervals must be removed before "
|
||||
"destroying a stream")
|
||||
self._get_intervals.cache_remove(self, stream_id)
|
||||
|
|
@ -473,7 +483,7 @@ class NilmDB():
|
|||
con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
|
||||
con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
|
||||
|
||||
def stream_insert(self, path, start, end, data, binary=False):
|
||||
def stream_insert(self, path, start, end, data, binary = False):
|
||||
"""Insert new data into the database.
|
||||
path: Path at which to add the data
|
||||
start: Starting timestamp
|
||||
|
|
@ -509,7 +519,7 @@ class NilmDB():
|
|||
# Like bisect.bisect_left, but doesn't choke on large indices on
|
||||
# 32-bit systems, like bisect's fast C implementation does.
|
||||
while lo < hi:
|
||||
mid = (lo + hi) // 2
|
||||
mid = (lo + hi) / 2
|
||||
if a[mid] < x:
|
||||
lo = mid + 1
|
||||
else:
|
||||
|
|
@ -550,8 +560,8 @@ class NilmDB():
|
|||
dbinterval.db_startpos,
|
||||
dbinterval.db_endpos)
|
||||
|
||||
def stream_extract(self, path, start=None, end=None,
|
||||
count=False, markup=False, binary=False):
|
||||
def stream_extract(self, path, start = None, end = None,
|
||||
count = False, markup = False, binary = False):
|
||||
"""
|
||||
Returns (data, restart) tuple.
|
||||
|
||||
|
|
@ -607,8 +617,8 @@ class NilmDB():
|
|||
|
||||
# Add markup
|
||||
if markup:
|
||||
result.append(b"# interval-start " +
|
||||
timestamp_to_bytes(interval.start) + b"\n")
|
||||
result.append("# interval-start " +
|
||||
timestamp_to_string(interval.start) + "\n")
|
||||
|
||||
# Gather these results up
|
||||
result.append(table.get_data(row_start, row_end, binary))
|
||||
|
|
@ -619,19 +629,18 @@ class NilmDB():
|
|||
# Add markup, and exit if restart is set.
|
||||
if restart is not None:
|
||||
if markup:
|
||||
result.append(b"# interval-end " +
|
||||
timestamp_to_bytes(restart) + b"\n")
|
||||
result.append("# interval-end " +
|
||||
timestamp_to_string(restart) + "\n")
|
||||
break
|
||||
if markup:
|
||||
result.append(b"# interval-end " +
|
||||
timestamp_to_bytes(interval.end) + b"\n")
|
||||
result.append("# interval-end " +
|
||||
timestamp_to_string(interval.end) + "\n")
|
||||
|
||||
if count:
|
||||
return matched
|
||||
full_result = b"".join(result)
|
||||
return (full_result, restart)
|
||||
return ("".join(result), restart)
|
||||
|
||||
def stream_remove(self, path, start=None, end=None):
|
||||
def stream_remove(self, path, start = None, end = None):
|
||||
"""
|
||||
Remove data from the specified time interval within a stream.
|
||||
|
||||
|
|
@ -658,7 +667,7 @@ class NilmDB():
|
|||
|
||||
# Can't remove intervals from within the iterator, so we need to
|
||||
# remember what's currently in the intersection now.
|
||||
all_candidates = list(intervals.intersection(to_remove, orig=True))
|
||||
all_candidates = list(intervals.intersection(to_remove, orig = True))
|
||||
|
||||
remove_start = None
|
||||
remove_end = None
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# cython: language_level=2
|
||||
|
||||
cdef class RBNode:
|
||||
cdef public object obj
|
||||
cdef public double start, end
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
# cython: profile=False
|
||||
# cython: cdivision=True
|
||||
# cython: language_level=2
|
||||
|
||||
"""
|
||||
Jim Paris <jim@jtan.com>
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ static void Rocket_dealloc(Rocket *self)
|
|||
fclose(self->file);
|
||||
self->file = NULL;
|
||||
}
|
||||
Py_TYPE(self)->tp_free((PyObject *)self);
|
||||
self->ob_type->tp_free((PyObject *)self);
|
||||
}
|
||||
|
||||
static PyObject *Rocket_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
|
||||
|
|
@ -160,19 +160,13 @@ static PyObject *Rocket_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
|
|||
static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
|
||||
{
|
||||
const char *layout, *path;
|
||||
int pathlen;
|
||||
static char *kwlist[] = { "layout", "file", NULL };
|
||||
if (!PyArg_ParseTupleAndKeywords(args, kwds, "sz#", kwlist,
|
||||
&layout, &path, &pathlen))
|
||||
if (!PyArg_ParseTupleAndKeywords(args, kwds, "sz", kwlist,
|
||||
&layout, &path))
|
||||
return -1;
|
||||
if (!layout)
|
||||
return -1;
|
||||
if (path) {
|
||||
if (strlen(path) != (size_t)pathlen) {
|
||||
PyErr_SetString(PyExc_ValueError, "path must not "
|
||||
"contain NUL characters");
|
||||
return -1;
|
||||
}
|
||||
if ((self->file = fopen(path, "a+b")) == NULL) {
|
||||
PyErr_SetFromErrno(PyExc_OSError);
|
||||
return -1;
|
||||
|
|
@ -245,7 +239,7 @@ static PyObject *Rocket_get_file_size(Rocket *self)
|
|||
return NULL;
|
||||
}
|
||||
}
|
||||
return PyLong_FromLong(self->file_size);
|
||||
return PyInt_FromLong(self->file_size);
|
||||
}
|
||||
|
||||
/****
|
||||
|
|
@ -279,9 +273,11 @@ static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
|
|||
union64_t t64;
|
||||
int i;
|
||||
|
||||
/* Input data is bytes. Using 'y#' instead of 'y' might be
|
||||
preferable, but strto* requires the null terminator. */
|
||||
if (!PyArg_ParseTuple(args, "iyiiLLL:append_string", &count,
|
||||
/* It would be nice to use 't#' instead of 's' for data,
|
||||
but we need the null termination for strto*. If we had
|
||||
strnto* that took a length, we could use t# and not require
|
||||
a copy. */
|
||||
if (!PyArg_ParseTuple(args, "isiiLLL:append_string", &count,
|
||||
&data, &offset, &linenum,
|
||||
&ll1, &ll2, &ll3))
|
||||
return NULL;
|
||||
|
|
@ -441,7 +437,7 @@ static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
|
|||
timestamp_t end;
|
||||
timestamp_t last_timestamp;
|
||||
|
||||
if (!PyArg_ParseTuple(args, "iy#iiLLL:append_binary",
|
||||
if (!PyArg_ParseTuple(args, "it#iiLLL:append_binary",
|
||||
&count, &data, &data_len, &offset,
|
||||
&linenum, &ll1, &ll2, &ll3))
|
||||
return NULL;
|
||||
|
|
@ -477,7 +473,7 @@ static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
|
|||
}
|
||||
|
||||
/* Write binary data */
|
||||
if (fwrite(data, self->binary_size, rows, self->file) != (size_t)rows) {
|
||||
if (fwrite(data, self->binary_size, rows, self->file) != rows) {
|
||||
PyErr_SetFromErrno(PyExc_OSError);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -491,7 +487,7 @@ static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
|
|||
}
|
||||
|
||||
/****
|
||||
* Extract to binary bytes object containing ASCII text-formatted data
|
||||
* Extract to string
|
||||
*/
|
||||
|
||||
static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
|
||||
|
|
@ -589,7 +585,7 @@ static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
|
|||
str[len++] = '\n';
|
||||
}
|
||||
|
||||
PyObject *pystr = PyBytes_FromStringAndSize(str, len);
|
||||
PyObject *pystr = PyString_FromStringAndSize(str, len);
|
||||
free(str);
|
||||
return pystr;
|
||||
err:
|
||||
|
|
@ -599,7 +595,7 @@ err:
|
|||
}
|
||||
|
||||
/****
|
||||
* Extract to binary bytes object containing raw little-endian binary data
|
||||
* Extract to binary string containing raw little-endian binary data
|
||||
*/
|
||||
static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args)
|
||||
{
|
||||
|
|
@ -628,7 +624,7 @@ static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args)
|
|||
|
||||
/* Data in the file is already in the desired little-endian
|
||||
binary format, so just read it directly. */
|
||||
if (fread(str, self->binary_size, count, self->file) != (size_t)count) {
|
||||
if (fread(str, self->binary_size, count, self->file) != count) {
|
||||
free(str);
|
||||
PyErr_SetFromErrno(PyExc_OSError);
|
||||
return NULL;
|
||||
|
|
@ -752,7 +748,7 @@ static PyMethodDef Rocket_methods[] = {
|
|||
};
|
||||
|
||||
static PyTypeObject RocketType = {
|
||||
PyVarObject_HEAD_INIT(NULL, 0)
|
||||
PyObject_HEAD_INIT(NULL)
|
||||
|
||||
.tp_name = "rocket.Rocket",
|
||||
.tp_basicsize = sizeof(Rocket),
|
||||
|
|
@ -777,23 +773,17 @@ static PyMethodDef module_methods[] = {
|
|||
{ NULL },
|
||||
};
|
||||
|
||||
static struct PyModuleDef moduledef = {
|
||||
PyModuleDef_HEAD_INIT,
|
||||
.m_name = "rocker",
|
||||
.m_doc = "Rocket data parsing and formatting module",
|
||||
.m_size = -1,
|
||||
.m_methods = module_methods,
|
||||
};
|
||||
|
||||
PyMODINIT_FUNC PyInit_rocket(void)
|
||||
PyMODINIT_FUNC
|
||||
initrocket(void)
|
||||
{
|
||||
PyObject *module;
|
||||
|
||||
RocketType.tp_new = PyType_GenericNew;
|
||||
if (PyType_Ready(&RocketType) < 0)
|
||||
return NULL;
|
||||
return;
|
||||
|
||||
module = PyModule_Create(&moduledef);
|
||||
module = Py_InitModule3("rocket", module_methods,
|
||||
"Rocket data parsing and formatting module");
|
||||
Py_INCREF(&RocketType);
|
||||
PyModule_AddObject(module, "Rocket", (PyObject *)&RocketType);
|
||||
|
||||
|
|
@ -802,5 +792,5 @@ PyMODINIT_FUNC PyInit_rocket(void)
|
|||
PyModule_AddObject(module, "ParseError", ParseError);
|
||||
add_parseerror_codes(module);
|
||||
|
||||
return module;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,26 @@
|
|||
"""CherryPy-based server for accessing NILM database via HTTP"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import socket
|
||||
import traceback
|
||||
|
||||
import psutil
|
||||
import cherrypy
|
||||
|
||||
# Need absolute_import so that "import nilmdb" won't pull in
|
||||
# nilmdb.py, but will pull the nilmdb module instead.
|
||||
from __future__ import absolute_import
|
||||
import nilmdb.server
|
||||
from nilmdb.utils.printf import sprintf
|
||||
from nilmdb.utils.printf import *
|
||||
from nilmdb.server.errors import NilmDBError
|
||||
from nilmdb.utils.time import string_to_timestamp
|
||||
|
||||
import cherrypy
|
||||
import sys
|
||||
import os
|
||||
import socket
|
||||
import simplejson as json
|
||||
import decorator
|
||||
import psutil
|
||||
import traceback
|
||||
|
||||
from nilmdb.server.serverutil import (
|
||||
chunked_response,
|
||||
response_type,
|
||||
workaround_cp_bug_1200,
|
||||
exception_to_httperror,
|
||||
CORS_allow,
|
||||
json_to_request_params,
|
||||
|
|
@ -28,15 +33,17 @@ from nilmdb.server.serverutil import (
|
|||
# Add CORS_allow tool
|
||||
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
|
||||
|
||||
|
||||
class NilmApp():
|
||||
class NilmApp(object):
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
|
||||
# CherryPy apps
|
||||
class Root(NilmApp):
|
||||
"""Root application for NILM database"""
|
||||
|
||||
def __init__(self, db):
|
||||
super(Root, self).__init__(db)
|
||||
|
||||
# /
|
||||
@cherrypy.expose
|
||||
def index(self):
|
||||
|
|
@ -65,14 +72,11 @@ class Root(NilmApp):
|
|||
path = self.db.get_basepath()
|
||||
usage = psutil.disk_usage(path)
|
||||
dbsize = nilmdb.utils.du(path)
|
||||
return {
|
||||
"path": path,
|
||||
return { "path": path,
|
||||
"size": dbsize,
|
||||
"other": max(usage.used - dbsize, 0),
|
||||
"reserved": max(usage.total - usage.used - usage.free, 0),
|
||||
"free": usage.free
|
||||
}
|
||||
|
||||
"free": usage.free }
|
||||
|
||||
class Stream(NilmApp):
|
||||
"""Stream-specific operations"""
|
||||
|
|
@ -85,8 +89,7 @@ class Stream(NilmApp):
|
|||
start = string_to_timestamp(start_param)
|
||||
except Exception:
|
||||
raise cherrypy.HTTPError("400 Bad Request", sprintf(
|
||||
"invalid start (%s): must be a numeric timestamp",
|
||||
start_param))
|
||||
"invalid start (%s): must be a numeric timestamp", start_param))
|
||||
try:
|
||||
if end_param is not None:
|
||||
end = string_to_timestamp(end_param)
|
||||
|
|
@ -106,7 +109,7 @@ class Stream(NilmApp):
|
|||
# /stream/list?path=/newton/prep&extended=1
|
||||
@cherrypy.expose
|
||||
@cherrypy.tools.json_out()
|
||||
def list(self, path=None, layout=None, extended=None):
|
||||
def list(self, path = None, layout = None, extended = None):
|
||||
"""List all streams in the database. With optional path or
|
||||
layout parameter, just list streams that match the given path
|
||||
or layout.
|
||||
|
|
@ -125,7 +128,7 @@ class Stream(NilmApp):
|
|||
@cherrypy.tools.json_in()
|
||||
@cherrypy.tools.json_out()
|
||||
@exception_to_httperror(NilmDBError, ValueError)
|
||||
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
||||
def create(self, path, layout):
|
||||
"""Create a new stream in the database. Provide path
|
||||
and one of the nilmdb.layout.layouts keys.
|
||||
|
|
@ -137,7 +140,7 @@ class Stream(NilmApp):
|
|||
@cherrypy.tools.json_in()
|
||||
@cherrypy.tools.json_out()
|
||||
@exception_to_httperror(NilmDBError)
|
||||
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
||||
def destroy(self, path):
|
||||
"""Delete a stream. Fails if any data is still present."""
|
||||
return self.db.stream_destroy(path)
|
||||
|
|
@ -147,7 +150,7 @@ class Stream(NilmApp):
|
|||
@cherrypy.tools.json_in()
|
||||
@cherrypy.tools.json_out()
|
||||
@exception_to_httperror(NilmDBError, ValueError)
|
||||
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
||||
def rename(self, oldpath, newpath):
|
||||
"""Rename a stream."""
|
||||
return self.db.stream_rename(oldpath, newpath)
|
||||
|
|
@ -163,11 +166,11 @@ class Stream(NilmApp):
|
|||
try:
|
||||
data = self.db.stream_get_metadata(path)
|
||||
except nilmdb.server.nilmdb.StreamError as e:
|
||||
raise cherrypy.HTTPError("404 Not Found", str(e))
|
||||
raise cherrypy.HTTPError("404 Not Found", e.message)
|
||||
if key is None: # If no keys specified, return them all
|
||||
key = list(data.keys())
|
||||
key = data.keys()
|
||||
elif not isinstance(key, list):
|
||||
key = [key]
|
||||
key = [ key ]
|
||||
result = {}
|
||||
for k in key:
|
||||
if k in data:
|
||||
|
|
@ -182,9 +185,11 @@ class Stream(NilmApp):
|
|||
try:
|
||||
data = dict(json.loads(data))
|
||||
except TypeError as e:
|
||||
raise NilmDBError("can't parse 'data' parameter: " + str(e))
|
||||
raise NilmDBError("can't parse 'data' parameter: " + e.message)
|
||||
for key in data:
|
||||
if not isinstance(data[key], (str, float, int)):
|
||||
if not (isinstance(data[key], basestring) or
|
||||
isinstance(data[key], float) or
|
||||
isinstance(data[key], int)):
|
||||
raise NilmDBError("metadata values must be a string or number")
|
||||
function(path, data)
|
||||
|
||||
|
|
@ -193,7 +198,7 @@ class Stream(NilmApp):
|
|||
@cherrypy.tools.json_in()
|
||||
@cherrypy.tools.json_out()
|
||||
@exception_to_httperror(NilmDBError, LookupError)
|
||||
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
||||
def set_metadata(self, path, data):
|
||||
"""Set metadata for the named stream, replacing any existing
|
||||
metadata. Data can be json-encoded or a plain dictionary."""
|
||||
|
|
@ -204,7 +209,7 @@ class Stream(NilmApp):
|
|||
@cherrypy.tools.json_in()
|
||||
@cherrypy.tools.json_out()
|
||||
@exception_to_httperror(NilmDBError, LookupError, ValueError)
|
||||
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
||||
def update_metadata(self, path, data):
|
||||
"""Set metadata for the named stream, replacing any existing
|
||||
metadata. Data can be json-encoded or a plain dictionary."""
|
||||
|
|
@ -214,8 +219,8 @@ class Stream(NilmApp):
|
|||
@cherrypy.expose
|
||||
@cherrypy.tools.json_out()
|
||||
@exception_to_httperror(NilmDBError, ValueError)
|
||||
@cherrypy.tools.CORS_allow(methods=["PUT"])
|
||||
def insert(self, path, start, end, binary=False):
|
||||
@cherrypy.tools.CORS_allow(methods = ["PUT"])
|
||||
def insert(self, path, start, end, binary = False):
|
||||
"""
|
||||
Insert new data into the database. Provide textual data
|
||||
(matching the path's layout) as a HTTP PUT.
|
||||
|
|
@ -241,11 +246,8 @@ class Stream(NilmApp):
|
|||
"application/octet-stream for "
|
||||
"binary data, not " + content_type)
|
||||
|
||||
# Note that non-binary data is *not* decoded from bytes to string,
|
||||
# but rather passed directly to stream_insert.
|
||||
|
||||
# Check path and get layout
|
||||
if len(self.db.stream_list(path=path)) != 1:
|
||||
if len(self.db.stream_list(path = path)) != 1:
|
||||
raise cherrypy.HTTPError("404", "No such stream: " + path)
|
||||
|
||||
# Check limits
|
||||
|
|
@ -262,10 +264,10 @@ class Stream(NilmApp):
|
|||
# /stream/remove?path=/newton/prep&start=1234567890.0&end=1234567899.0
|
||||
@cherrypy.expose
|
||||
@cherrypy.tools.json_in()
|
||||
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
||||
@chunked_response
|
||||
@response_type("application/x-json-stream")
|
||||
def remove(self, path, start=None, end=None):
|
||||
def remove(self, path, start = None, end = None):
|
||||
"""
|
||||
Remove data from the backend database. Removes all data in
|
||||
the interval [start, end).
|
||||
|
|
@ -277,15 +279,15 @@ class Stream(NilmApp):
|
|||
"""
|
||||
(start, end) = self._get_times(start, end)
|
||||
|
||||
if len(self.db.stream_list(path=path)) != 1:
|
||||
if len(self.db.stream_list(path = path)) != 1:
|
||||
raise cherrypy.HTTPError("404", "No such stream: " + path)
|
||||
|
||||
@workaround_cp_bug_1200
|
||||
def content(start, end):
|
||||
# Note: disable chunked responses to see tracebacks from here.
|
||||
while True:
|
||||
(removed, restart) = self.db.stream_remove(path, start, end)
|
||||
response = json.dumps(removed) + "\r\n"
|
||||
yield response.encode('utf-8')
|
||||
yield json.dumps(removed) + "\r\n"
|
||||
if restart is None:
|
||||
break
|
||||
start = restart
|
||||
|
|
@ -297,7 +299,7 @@ class Stream(NilmApp):
|
|||
@cherrypy.expose
|
||||
@chunked_response
|
||||
@response_type("application/x-json-stream")
|
||||
def intervals(self, path, start=None, end=None, diffpath=None):
|
||||
def intervals(self, path, start = None, end = None, diffpath = None):
|
||||
"""
|
||||
Get intervals from backend database. Streams the resulting
|
||||
intervals as JSON strings separated by CR LF pairs. This may
|
||||
|
|
@ -314,19 +316,20 @@ class Stream(NilmApp):
|
|||
"""
|
||||
(start, end) = self._get_times(start, end)
|
||||
|
||||
if len(self.db.stream_list(path=path)) != 1:
|
||||
if len(self.db.stream_list(path = path)) != 1:
|
||||
raise cherrypy.HTTPError("404", "No such stream: " + path)
|
||||
|
||||
if diffpath and len(self.db.stream_list(path=diffpath)) != 1:
|
||||
if diffpath and len(self.db.stream_list(path = diffpath)) != 1:
|
||||
raise cherrypy.HTTPError("404", "No such stream: " + diffpath)
|
||||
|
||||
@workaround_cp_bug_1200
|
||||
def content(start, end):
|
||||
# Note: disable chunked responses to see tracebacks from here.
|
||||
while True:
|
||||
(ints, restart) = self.db.stream_intervals(path, start, end,
|
||||
diffpath)
|
||||
response = ''.join([json.dumps(i) + "\r\n" for i in ints])
|
||||
yield response.encode('utf-8')
|
||||
response = ''.join([ json.dumps(i) + "\r\n" for i in ints ])
|
||||
yield response
|
||||
if restart is None:
|
||||
break
|
||||
start = restart
|
||||
|
|
@ -335,8 +338,8 @@ class Stream(NilmApp):
|
|||
# /stream/extract?path=/newton/prep&start=1234567890.0&end=1234567899.0
|
||||
@cherrypy.expose
|
||||
@chunked_response
|
||||
def extract(self, path, start=None, end=None,
|
||||
count=False, markup=False, binary=False):
|
||||
def extract(self, path, start = None, end = None,
|
||||
count = False, markup = False, binary = False):
|
||||
"""
|
||||
Extract data from backend database. Streams the resulting
|
||||
entries as ASCII text lines separated by newlines. This may
|
||||
|
|
@ -360,7 +363,7 @@ class Stream(NilmApp):
|
|||
(start, end) = self._get_times(start, end)
|
||||
|
||||
# Check path and get layout
|
||||
if len(self.db.stream_list(path=path)) != 1:
|
||||
if len(self.db.stream_list(path = path)) != 1:
|
||||
raise cherrypy.HTTPError("404", "No such stream: " + path)
|
||||
|
||||
if binary:
|
||||
|
|
@ -372,18 +375,19 @@ class Stream(NilmApp):
|
|||
content_type = "text/plain"
|
||||
cherrypy.response.headers['Content-Type'] = content_type
|
||||
|
||||
@workaround_cp_bug_1200
|
||||
def content(start, end):
|
||||
# Note: disable chunked responses to see tracebacks from here.
|
||||
if count:
|
||||
matched = self.db.stream_extract(path, start, end,
|
||||
count=True)
|
||||
yield sprintf(b"%d\n", matched)
|
||||
count = True)
|
||||
yield sprintf("%d\n", matched)
|
||||
return
|
||||
|
||||
while True:
|
||||
(data, restart) = self.db.stream_extract(
|
||||
path, start, end, count=False,
|
||||
markup=markup, binary=binary)
|
||||
path, start, end, count = False,
|
||||
markup = markup, binary = binary)
|
||||
yield data
|
||||
|
||||
if restart is None:
|
||||
|
|
@ -391,31 +395,29 @@ class Stream(NilmApp):
|
|||
start = restart
|
||||
return content(start, end)
|
||||
|
||||
|
||||
class Exiter():
|
||||
class Exiter(object):
|
||||
"""App that exits the server, for testing"""
|
||||
@cherrypy.expose
|
||||
def index(self):
|
||||
cherrypy.response.headers['Content-Type'] = 'text/plain'
|
||||
|
||||
def content():
|
||||
yield b'Exiting by request'
|
||||
yield 'Exiting by request'
|
||||
raise SystemExit
|
||||
|
||||
return content()
|
||||
index._cp_config = {'response.stream': True}
|
||||
index._cp_config = { 'response.stream': True }
|
||||
|
||||
|
||||
class Server():
|
||||
def __init__(self, db, host='127.0.0.1', port=8080,
|
||||
stoppable=False, # whether /exit URL exists
|
||||
fast_shutdown=False, # don't wait for clients to disconn.
|
||||
force_traceback=False, # include traceback in all errors
|
||||
basepath='', # base URL path for cherrypy.tree
|
||||
class Server(object):
|
||||
def __init__(self, db, host = '127.0.0.1', port = 8080,
|
||||
stoppable = False, # whether /exit URL exists
|
||||
embedded = True, # hide diagnostics and output, etc
|
||||
fast_shutdown = False, # don't wait for clients to disconn.
|
||||
force_traceback = False, # include traceback in all errors
|
||||
basepath = '', # base URL path for cherrypy.tree
|
||||
):
|
||||
# Save server version, just for verification during tests
|
||||
self.version = nilmdb.__version__
|
||||
|
||||
self.embedded = embedded
|
||||
self.db = db
|
||||
if not getattr(db, "_thread_safe", None):
|
||||
raise KeyError("Database object " + str(db) + " doesn't claim "
|
||||
|
|
@ -425,12 +427,13 @@ class Server():
|
|||
|
||||
# Build up global server configuration
|
||||
cherrypy.config.update({
|
||||
'environment': 'embedded',
|
||||
'server.socket_host': host,
|
||||
'server.socket_port': port,
|
||||
'engine.autoreload.on': False,
|
||||
'server.max_request_body_size': 8*1024*1024,
|
||||
})
|
||||
if self.embedded:
|
||||
cherrypy.config.update({ 'environment': 'embedded' })
|
||||
|
||||
# Build up application specific configuration
|
||||
app_config = {}
|
||||
|
|
@ -439,23 +442,23 @@ class Server():
|
|||
})
|
||||
|
||||
# Some default headers to just help identify that things are working
|
||||
app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
|
||||
app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
|
||||
|
||||
# Set up Cross-Origin Resource Sharing (CORS) handler so we
|
||||
# can correctly respond to browsers' CORS preflight requests.
|
||||
# This also limits verbs to GET and HEAD by default.
|
||||
app_config.update({'tools.CORS_allow.on': True,
|
||||
'tools.CORS_allow.methods': ['GET', 'HEAD']})
|
||||
app_config.update({ 'tools.CORS_allow.on': True,
|
||||
'tools.CORS_allow.methods': ['GET', 'HEAD'] })
|
||||
|
||||
# Configure the 'json_in' tool to also allow other content-types
|
||||
# (like x-www-form-urlencoded), and to treat JSON as a dict that
|
||||
# fills requests.param.
|
||||
app_config.update({'tools.json_in.force': False,
|
||||
'tools.json_in.processor': json_to_request_params})
|
||||
app_config.update({ 'tools.json_in.force': False,
|
||||
'tools.json_in.processor': json_to_request_params })
|
||||
|
||||
# Send tracebacks in error responses. They're hidden by the
|
||||
# error_page function for client errors (code 400-499).
|
||||
app_config.update({'request.show_tracebacks': True})
|
||||
app_config.update({ 'request.show_tracebacks' : True })
|
||||
self.force_traceback = force_traceback
|
||||
|
||||
# Patch CherryPy error handler to never pad out error messages.
|
||||
|
|
@ -469,12 +472,13 @@ class Server():
|
|||
if stoppable:
|
||||
root.exit = Exiter()
|
||||
cherrypy.tree.apps = {}
|
||||
cherrypy.tree.mount(root, basepath, config={"/": app_config})
|
||||
cherrypy.tree.mount(root, basepath, config = { "/" : app_config })
|
||||
|
||||
# Shutdowns normally wait for clients to disconnect. To speed
|
||||
# up tests, set fast_shutdown = True
|
||||
if fast_shutdown:
|
||||
cherrypy.server.shutdown_timeout = 0
|
||||
# Setting timeout to 0 triggers os._exit(70) at shutdown, grr...
|
||||
cherrypy.server.shutdown_timeout = 0.01
|
||||
else:
|
||||
cherrypy.server.shutdown_timeout = 5
|
||||
|
||||
|
|
@ -486,21 +490,18 @@ class Server():
|
|||
return json_error_page(status, message, traceback, version,
|
||||
self.force_traceback)
|
||||
|
||||
def start(self, blocking=False, event=None):
|
||||
cherrypy_start(blocking, event)
|
||||
def start(self, blocking = False, event = None):
|
||||
cherrypy_start(blocking, event, self.embedded)
|
||||
|
||||
def stop(self):
|
||||
cherrypy_stop()
|
||||
|
||||
|
||||
# Use a single global nilmdb.server.NilmDB and nilmdb.server.Server
|
||||
# instance since the database can only be opened once. For this to
|
||||
# work, the web server must use only a single process and single
|
||||
# Python interpreter. Multiple threads are OK.
|
||||
_wsgi_server = None
|
||||
|
||||
|
||||
def wsgi_application(dbpath, basepath):
|
||||
def wsgi_application(dbpath, basepath): # pragma: no cover
|
||||
"""Return a WSGI application object with a database at the
|
||||
specified path.
|
||||
|
||||
|
|
@ -515,16 +516,17 @@ def wsgi_application(dbpath, basepath):
|
|||
if _wsgi_server is None:
|
||||
# Try to start the server
|
||||
try:
|
||||
db = nilmdb.utils.serializer_proxy(
|
||||
nilmdb.server.NilmDB)(dbpath)
|
||||
db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(dbpath)
|
||||
_wsgi_server = nilmdb.server.Server(
|
||||
db, basepath=basepath.rstrip('/'))
|
||||
db, embedded = True,
|
||||
basepath = basepath.rstrip('/'))
|
||||
except Exception:
|
||||
# Build an error message on failure
|
||||
import pprint
|
||||
err = sprintf("Initializing database at path '%s' failed:\n\n",
|
||||
dbpath)
|
||||
err += traceback.format_exc()
|
||||
try:
|
||||
import pwd
|
||||
import grp
|
||||
err += sprintf("\nRunning as: uid=%d (%s), gid=%d (%s) "
|
||||
|
|
@ -532,14 +534,15 @@ def wsgi_application(dbpath, basepath):
|
|||
os.getuid(), pwd.getpwuid(os.getuid())[0],
|
||||
os.getgid(), grp.getgrgid(os.getgid())[0],
|
||||
socket.gethostname(), os.getpid())
|
||||
except ImportError:
|
||||
pass
|
||||
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
|
||||
if _wsgi_server is None:
|
||||
# Serve up the error with our own mini WSGI app.
|
||||
err_b = err.encode('utf-8')
|
||||
headers = [('Content-type', 'text/plain; charset=utf-8'),
|
||||
('Content-length', str(len(err_b)))]
|
||||
headers = [ ('Content-type', 'text/plain'),
|
||||
('Content-length', str(len(err))) ]
|
||||
start_response("500 Internal Server Error", headers)
|
||||
return [err_b]
|
||||
return [err]
|
||||
|
||||
# Call the normal application
|
||||
return _wsgi_server.wsgi_application(environ, start_response)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,11 @@
|
|||
"""Miscellaneous decorators and other helpers for running a CherryPy
|
||||
server"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import decorator
|
||||
import functools
|
||||
import threading
|
||||
|
||||
import cherrypy
|
||||
|
||||
import sys
|
||||
import os
|
||||
import decorator
|
||||
import simplejson as json
|
||||
|
||||
# Helper to parse parameters into booleans
|
||||
def bool_param(s):
|
||||
|
|
@ -17,25 +13,23 @@ def bool_param(s):
|
|||
supporting a few different types for 's'."""
|
||||
try:
|
||||
ss = s.lower()
|
||||
if ss in ["0", "false", "f", "no", "n"]:
|
||||
if ss in [ "0", "false", "f", "no", "n" ]:
|
||||
return False
|
||||
if ss in ["1", "true", "t", "yes", "y"]:
|
||||
if ss in [ "1", "true", "t", "yes", "y" ]:
|
||||
return True
|
||||
except Exception:
|
||||
return bool(s)
|
||||
raise cherrypy.HTTPError("400 Bad Request",
|
||||
"can't parse parameter: " + ss)
|
||||
|
||||
|
||||
# Decorators
|
||||
def chunked_response(func):
|
||||
"""Decorator to enable chunked responses."""
|
||||
# Set this to False to get better tracebacks from some requests
|
||||
# (/stream/extract, /stream/intervals).
|
||||
func._cp_config = {'response.stream': True}
|
||||
func._cp_config = { 'response.stream': True }
|
||||
return func
|
||||
|
||||
|
||||
def response_type(content_type):
|
||||
"""Return a decorator-generating function that sets the
|
||||
response type to the specified string."""
|
||||
|
|
@ -44,6 +38,27 @@ def response_type(content_type):
|
|||
return func(*args, **kwargs)
|
||||
return decorator.decorator(wrapper)
|
||||
|
||||
@decorator.decorator
|
||||
def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
|
||||
"""Decorator to work around CherryPy bug #1200 in a response
|
||||
generator.
|
||||
|
||||
Even if chunked responses are disabled, LookupError or
|
||||
UnicodeError exceptions may still be swallowed by CherryPy due to
|
||||
bug #1200. This throws them as generic Exceptions instead so that
|
||||
they make it through.
|
||||
"""
|
||||
exc_info = None
|
||||
try:
|
||||
for val in func(*args, **kwargs):
|
||||
yield val
|
||||
except (LookupError, UnicodeError):
|
||||
# Re-raise it, but maintain the original traceback
|
||||
exc_info = sys.exc_info()
|
||||
new_exc = Exception(exc_info[0].__name__ + ": " + str(exc_info[1]))
|
||||
raise new_exc, None, exc_info[2]
|
||||
finally:
|
||||
del exc_info
|
||||
|
||||
def exception_to_httperror(*expected):
|
||||
"""Return a decorator-generating function that catches expected
|
||||
|
|
@ -61,7 +76,7 @@ def exception_to_httperror(*expected):
|
|||
# Re-raise it, but maintain the original traceback
|
||||
exc_info = sys.exc_info()
|
||||
new_exc = cherrypy.HTTPError("400 Bad Request", str(exc_info[1]))
|
||||
raise new_exc.with_traceback(exc_info[2])
|
||||
raise new_exc, None, exc_info[2]
|
||||
finally:
|
||||
del exc_info
|
||||
# We need to preserve the function's argspecs for CherryPy to
|
||||
|
|
@ -69,8 +84,8 @@ def exception_to_httperror(*expected):
|
|||
# care of that.
|
||||
return decorator.decorator(wrapper)
|
||||
|
||||
|
||||
# Custom CherryPy tools
|
||||
|
||||
def CORS_allow(methods):
|
||||
"""This does several things:
|
||||
|
||||
|
|
@ -86,12 +101,12 @@ def CORS_allow(methods):
|
|||
request = cherrypy.request.headers
|
||||
response = cherrypy.response.headers
|
||||
|
||||
if not isinstance(methods, (tuple, list)):
|
||||
methods = [methods]
|
||||
methods = [m.upper() for m in methods if m]
|
||||
if not methods:
|
||||
methods = ['GET', 'HEAD']
|
||||
elif 'GET' in methods and 'HEAD' not in methods:
|
||||
if not isinstance(methods, (tuple, list)): # pragma: no cover
|
||||
methods = [ methods ]
|
||||
methods = [ m.upper() for m in methods if m ]
|
||||
if not methods: # pragma: no cover
|
||||
methods = [ 'GET', 'HEAD' ]
|
||||
elif 'GET' in methods and 'HEAD' not in methods: # pragma: no cover
|
||||
methods.append('HEAD')
|
||||
response['Allow'] = ', '.join(methods)
|
||||
|
||||
|
|
@ -108,7 +123,7 @@ def CORS_allow(methods):
|
|||
response['Access-Control-Allow-Methods'] = ', '.join(methods)
|
||||
# Try to stop further processing and return a 200 OK
|
||||
cherrypy.response.status = "200 OK"
|
||||
cherrypy.response.body = b""
|
||||
cherrypy.response.body = ""
|
||||
cherrypy.request.handler = lambda: ""
|
||||
return
|
||||
|
||||
|
|
@ -125,83 +140,58 @@ def json_to_request_params(body):
|
|||
raise cherrypy.HTTPError(415)
|
||||
cherrypy.request.params.update(cherrypy.request.json)
|
||||
|
||||
|
||||
# Used as an "error_page.default" handler
|
||||
def json_error_page(status, message, traceback, version,
|
||||
force_traceback=False):
|
||||
force_traceback = False):
|
||||
"""Return a custom error page in JSON so the client can parse it"""
|
||||
errordata = {"status": status,
|
||||
"message": message,
|
||||
"version": version,
|
||||
"traceback": traceback}
|
||||
errordata = { "status" : status,
|
||||
"message" : message,
|
||||
"traceback" : traceback }
|
||||
# Don't send a traceback if the error was 400-499 (client's fault)
|
||||
try:
|
||||
code = int(status.split()[0])
|
||||
if not force_traceback:
|
||||
if 400 <= code <= 499:
|
||||
if code >= 400 and code <= 499:
|
||||
errordata["traceback"] = ""
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
# Override the response type, which was previously set to text/html
|
||||
cherrypy.serving.response.headers['Content-Type'] = (
|
||||
"application/json;charset=utf-8")
|
||||
"application/json;charset=utf-8" )
|
||||
# Undo the HTML escaping that cherrypy's get_error_page function applies
|
||||
# (cherrypy issue 1135)
|
||||
for k, v in errordata.items():
|
||||
v = v.replace("<", "<")
|
||||
v = v.replace(">", ">")
|
||||
v = v.replace("&", "&")
|
||||
for k, v in errordata.iteritems():
|
||||
v = v.replace("<","<")
|
||||
v = v.replace(">",">")
|
||||
v = v.replace("&","&")
|
||||
errordata[k] = v
|
||||
return json.dumps(errordata, separators=(',', ':'))
|
||||
|
||||
|
||||
class CherryPyExit(SystemExit):
|
||||
pass
|
||||
|
||||
|
||||
def cherrypy_patch_exit():
|
||||
# Cherrypy stupidly calls os._exit(70) when it can't bind the port
|
||||
# and exits. Instead of that, raise a CherryPyExit (derived from
|
||||
# SystemExit). This exception may not make it back up to the caller
|
||||
# due to internal thread use in the CherryPy engine, but there should
|
||||
# be at least some indication that it happened.
|
||||
bus = cherrypy.process.wspbus.bus
|
||||
if "_patched_exit" in bus.__dict__:
|
||||
return
|
||||
bus._patched_exit = True
|
||||
|
||||
def patched_exit(orig):
|
||||
real_exit = os._exit
|
||||
|
||||
def fake_exit(code):
|
||||
raise CherryPyExit(code)
|
||||
os._exit = fake_exit
|
||||
try:
|
||||
orig()
|
||||
finally:
|
||||
os._exit = real_exit
|
||||
bus.exit = functools.partial(patched_exit, bus.exit)
|
||||
|
||||
# A behavior change in Python 3.8 means that some thread exceptions,
|
||||
# derived from SystemExit, now print tracebacks where they didn't
|
||||
# used to: https://bugs.python.org/issue1230540
|
||||
# Install a thread exception hook that ignores CherryPyExit;
|
||||
# to make this match the behavior where we didn't set
|
||||
# threading.excepthook, we also need to ignore SystemExit.
|
||||
def hook(args):
|
||||
if args.exc_type == CherryPyExit or args.exc_type == SystemExit:
|
||||
return
|
||||
sys.excepthook(args.exc_type, args.exc_value,
|
||||
args.exc_traceback) # pragma: no cover
|
||||
threading.excepthook = hook
|
||||
|
||||
return json.dumps(errordata, separators=(',',':'))
|
||||
|
||||
# Start/stop CherryPy standalone server
|
||||
def cherrypy_start(blocking=False, event=False):
|
||||
def cherrypy_start(blocking = False, event = False, embedded = False):
|
||||
"""Start the CherryPy server, handling errors and signals
|
||||
somewhat gracefully."""
|
||||
|
||||
cherrypy_patch_exit()
|
||||
if not embedded: # pragma: no cover
|
||||
# Handle signals nicely
|
||||
if hasattr(cherrypy.engine, "signal_handler"):
|
||||
cherrypy.engine.signal_handler.subscribe()
|
||||
if hasattr(cherrypy.engine, "console_control_handler"):
|
||||
cherrypy.engine.console_control_handler.subscribe()
|
||||
|
||||
# Start the server
|
||||
# Cherrypy stupidly calls os._exit(70) when it can't bind the
|
||||
# port. At least try to print a reasonable error and continue
|
||||
# in this case, rather than just dying silently (as we would
|
||||
# otherwise do in embedded mode)
|
||||
real_exit = os._exit
|
||||
def fake_exit(code): # pragma: no cover
|
||||
if code == os.EX_SOFTWARE:
|
||||
fprintf(sys.stderr, "error: CherryPy called os._exit!\n")
|
||||
else:
|
||||
real_exit(code)
|
||||
os._exit = fake_exit
|
||||
cherrypy.engine.start()
|
||||
os._exit = real_exit
|
||||
|
||||
# Signal that the engine has started successfully
|
||||
if event is not None:
|
||||
|
|
@ -210,16 +200,15 @@ def cherrypy_start(blocking=False, event=False):
|
|||
if blocking:
|
||||
try:
|
||||
cherrypy.engine.wait(cherrypy.engine.states.EXITING,
|
||||
interval=0.1, channel='main')
|
||||
except (KeyboardInterrupt, IOError):
|
||||
cherrypy.engine.log('Keyboard Interrupt: shutting down')
|
||||
interval = 0.1, channel = 'main')
|
||||
except (KeyboardInterrupt, IOError): # pragma: no cover
|
||||
cherrypy.engine.log('Keyboard Interrupt: shutting down bus')
|
||||
cherrypy.engine.exit()
|
||||
except SystemExit:
|
||||
cherrypy.engine.log('SystemExit raised: shutting down')
|
||||
except SystemExit: # pragma: no cover
|
||||
cherrypy.engine.log('SystemExit raised: shutting down bus')
|
||||
cherrypy.engine.exit()
|
||||
raise
|
||||
|
||||
|
||||
# Stop CherryPy server
|
||||
def cherrypy_stop():
|
||||
cherrypy.engine.exit()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""NilmDB utilities"""
|
||||
|
||||
|
||||
from __future__ import absolute_import
|
||||
from nilmdb.utils.timer import Timer
|
||||
from nilmdb.utils.serializer import serializer_proxy
|
||||
from nilmdb.utils.lrucache import lru_cache
|
||||
|
|
@ -14,3 +14,4 @@ import nilmdb.utils.iterator
|
|||
import nilmdb.utils.interval
|
||||
import nilmdb.utils.lock
|
||||
import nilmdb.utils.sort
|
||||
import nilmdb.utils.unicode
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
|
||||
import os
|
||||
|
||||
|
||||
def replace_file(filename, content):
|
||||
"""Attempt to atomically and durably replace the filename with the
|
||||
given contents"""
|
||||
given contents. This is intended to be 'pretty good on most
|
||||
OSes', but not necessarily bulletproof."""
|
||||
|
||||
newfilename = filename + b".new"
|
||||
newfilename = filename + ".new"
|
||||
|
||||
# Write to new file, flush it
|
||||
with open(newfilename, "wb") as f:
|
||||
|
|
@ -16,4 +16,11 @@ def replace_file(filename, content):
|
|||
os.fsync(f.fileno())
|
||||
|
||||
# Move new file over old one
|
||||
os.replace(newfilename, filename)
|
||||
try:
|
||||
os.rename(newfilename, filename)
|
||||
except OSError: # pragma: no cover
|
||||
# Some OSes might not support renaming over an existing file.
|
||||
# This is definitely NOT atomic!
|
||||
os.remove(filename)
|
||||
os.rename(newfilename, filename)
|
||||
|
||||
|
|
|
|||
710
nilmdb/utils/datetime_tz/__init__.py
Normal file
710
nilmdb/utils/datetime_tz/__init__.py
Normal file
|
|
@ -0,0 +1,710 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# Copyright 2009 Google Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
#
|
||||
# Disable the invalid name warning as we are inheriting from a standard library
|
||||
# object.
|
||||
# pylint: disable-msg=C6409,W0212
|
||||
|
||||
"""A version of the datetime module which *cares* about timezones.
|
||||
|
||||
This module will never return a naive datetime object. This requires the module
|
||||
know your local timezone, which it tries really hard to figure out.
|
||||
|
||||
You can override the detection by using the datetime.tzaware.defaulttz_set
|
||||
method. It the module is unable to figure out the timezone itself this method
|
||||
*must* be called before the normal module is imported. If done before importing
|
||||
it can also speed up the time taken to import as the defaulttz will no longer
|
||||
try and do the detection.
|
||||
"""
|
||||
|
||||
__author__ = "tansell@google.com (Tim Ansell)"
|
||||
|
||||
import calendar
|
||||
import datetime
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import time
|
||||
import warnings
|
||||
import dateutil.parser
|
||||
import dateutil.relativedelta
|
||||
import dateutil.tz
|
||||
import pytz
|
||||
import pytz_abbr
|
||||
|
||||
|
||||
try:
|
||||
# pylint: disable-msg=C6204
|
||||
import functools
|
||||
except ImportError, e:
|
||||
|
||||
class functools(object):
|
||||
"""Fake replacement for a full functools."""
|
||||
|
||||
# pylint: disable-msg=W0613
|
||||
@staticmethod
|
||||
def wraps(f, *args, **kw):
|
||||
return f
|
||||
|
||||
|
||||
# Need to patch pytz.utc to have a _utcoffset so you can normalize/localize
|
||||
# using it.
|
||||
pytz.utc._utcoffset = datetime.timedelta()
|
||||
|
||||
|
||||
timedelta = datetime.timedelta
|
||||
|
||||
|
||||
def _tzinfome(tzinfo):
|
||||
"""Gets a tzinfo object from a string.
|
||||
|
||||
Args:
|
||||
tzinfo: A string (or string like) object, or a datetime.tzinfo object.
|
||||
|
||||
Returns:
|
||||
An datetime.tzinfo object.
|
||||
|
||||
Raises:
|
||||
UnknownTimeZoneError: If the timezone given can't be decoded.
|
||||
"""
|
||||
if not isinstance(tzinfo, datetime.tzinfo):
|
||||
try:
|
||||
tzinfo = pytz.timezone(tzinfo)
|
||||
except AttributeError:
|
||||
raise pytz.UnknownTimeZoneError("Unknown timezone! %s" % tzinfo)
|
||||
return tzinfo
|
||||
|
||||
|
||||
# Our "local" timezone
|
||||
_localtz = None
|
||||
|
||||
|
||||
def localtz():
|
||||
"""Get the local timezone.
|
||||
|
||||
Returns:
|
||||
The localtime timezone as a tzinfo object.
|
||||
"""
|
||||
# pylint: disable-msg=W0603
|
||||
global _localtz
|
||||
if _localtz is None:
|
||||
_localtz = detect_timezone()
|
||||
return _localtz
|
||||
|
||||
|
||||
def localtz_set(timezone):
|
||||
"""Set the local timezone."""
|
||||
# pylint: disable-msg=W0603
|
||||
global _localtz
|
||||
_localtz = _tzinfome(timezone)
|
||||
|
||||
|
||||
def detect_timezone():
|
||||
"""Try and detect the timezone that Python is currently running in.
|
||||
|
||||
We have a bunch of different methods for trying to figure this out (listed in
|
||||
order they are attempted).
|
||||
* Try TZ environment variable.
|
||||
* Try and find /etc/timezone file (with timezone name).
|
||||
* Try and find /etc/localtime file (with timezone data).
|
||||
* Try and match a TZ to the current dst/offset/shortname.
|
||||
|
||||
Returns:
|
||||
The detected local timezone as a tzinfo object
|
||||
|
||||
Raises:
|
||||
pytz.UnknownTimeZoneError: If it was unable to detect a timezone.
|
||||
"""
|
||||
# First we try the TZ variable
|
||||
tz = _detect_timezone_environ()
|
||||
if tz is not None:
|
||||
return tz
|
||||
|
||||
# Second we try /etc/timezone and use the value in that
|
||||
tz = _detect_timezone_etc_timezone()
|
||||
if tz is not None:
|
||||
return tz
|
||||
|
||||
# Next we try and see if something matches the tzinfo in /etc/localtime
|
||||
tz = _detect_timezone_etc_localtime()
|
||||
if tz is not None:
|
||||
return tz
|
||||
|
||||
# Next we try and use a similiar method to what PHP does.
|
||||
# We first try to search on time.tzname, time.timezone, time.daylight to
|
||||
# match a pytz zone.
|
||||
warnings.warn("Had to fall back to worst detection method (the 'PHP' "
|
||||
"method).")
|
||||
|
||||
tz = _detect_timezone_php()
|
||||
if tz is not None:
|
||||
return tz
|
||||
|
||||
raise pytz.UnknownTimeZoneError("Unable to detect your timezone!")
|
||||
|
||||
|
||||
def _detect_timezone_environ():
|
||||
if "TZ" in os.environ:
|
||||
try:
|
||||
return pytz.timezone(os.environ["TZ"])
|
||||
except (IOError, pytz.UnknownTimeZoneError):
|
||||
warnings.warn("You provided a TZ environment value (%r) we did not "
|
||||
"understand!" % os.environ["TZ"])
|
||||
|
||||
|
||||
def _detect_timezone_etc_timezone():
|
||||
if os.path.exists("/etc/timezone"):
|
||||
try:
|
||||
tz = file("/etc/timezone").read().strip()
|
||||
try:
|
||||
return pytz.timezone(tz)
|
||||
except (IOError, pytz.UnknownTimeZoneError), ei:
|
||||
warnings.warn("Your /etc/timezone file references a timezone (%r) that"
|
||||
" is not valid (%r)." % (tz, ei))
|
||||
|
||||
# Problem reading the /etc/timezone file
|
||||
except IOError, eo:
|
||||
warnings.warn("Could not access your /etc/timezone file: %s" % eo)
|
||||
|
||||
|
||||
def _detect_timezone_etc_localtime():
|
||||
matches = []
|
||||
if os.path.exists("/etc/localtime"):
|
||||
localtime = pytz.tzfile.build_tzinfo("/etc/localtime",
|
||||
file("/etc/localtime"))
|
||||
|
||||
# See if we can find a "Human Name" for this..
|
||||
for tzname in pytz.all_timezones:
|
||||
tz = _tzinfome(tzname)
|
||||
|
||||
if dir(tz) != dir(localtime):
|
||||
continue
|
||||
|
||||
for attrib in dir(tz):
|
||||
# Ignore functions and specials
|
||||
if callable(getattr(tz, attrib)) or attrib.startswith("__"):
|
||||
continue
|
||||
|
||||
# This will always be different
|
||||
if attrib == "zone" or attrib == "_tzinfos":
|
||||
continue
|
||||
|
||||
if getattr(tz, attrib) != getattr(localtime, attrib):
|
||||
break
|
||||
|
||||
# We get here iff break didn't happen, i.e. no meaningful attributes
|
||||
# differ between tz and localtime
|
||||
else:
|
||||
matches.append(tzname)
|
||||
|
||||
if len(matches) == 1:
|
||||
return _tzinfome(matches[0])
|
||||
else:
|
||||
# Warn the person about this!
|
||||
warning = "Could not get a human name for your timezone: "
|
||||
if len(matches) > 1:
|
||||
warning += ("We detected multiple matches for your /etc/localtime. "
|
||||
"(Matches where %s)" % matches)
|
||||
return _tzinfome(matches[0])
|
||||
else:
|
||||
warning += "We detected no matches for your /etc/localtime."
|
||||
warnings.warn(warning)
|
||||
|
||||
# Register /etc/localtime as the timezone loaded.
|
||||
pytz._tzinfo_cache['/etc/localtime'] = localtime
|
||||
return localtime
|
||||
|
||||
|
||||
def _detect_timezone_php():
|
||||
tomatch = (time.tzname[0], time.timezone, time.daylight)
|
||||
now = datetime.datetime.now()
|
||||
|
||||
matches = []
|
||||
for tzname in pytz.all_timezones:
|
||||
try:
|
||||
tz = pytz.timezone(tzname)
|
||||
except IOError:
|
||||
continue
|
||||
|
||||
try:
|
||||
indst = tz.localize(now).timetuple()[-1]
|
||||
|
||||
if tomatch == (tz._tzname, -tz._utcoffset.seconds, indst):
|
||||
matches.append(tzname)
|
||||
|
||||
# pylint: disable-msg=W0704
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
if len(matches) > 1:
|
||||
warnings.warn("We detected multiple matches for the timezone, choosing "
|
||||
"the first %s. (Matches where %s)" % (matches[0], matches))
|
||||
return pytz.timezone(matches[0])
|
||||
|
||||
|
||||
class datetime_tz(datetime.datetime):
|
||||
"""An extension of the inbuilt datetime adding more functionality.
|
||||
|
||||
The extra functionality includes:
|
||||
* Partial parsing support (IE 2006/02/30 matches %Y/%M/%D %H:%M)
|
||||
* Full integration with pytz (just give it the string of the timezone!)
|
||||
* Proper support for going to/from Unix timestamps (which are in UTC!).
|
||||
"""
|
||||
__slots__ = ["is_dst"]
|
||||
|
||||
def __new__(cls, *args, **kw):
|
||||
args = list(args)
|
||||
if not args:
|
||||
raise TypeError("Not enough arguments given.")
|
||||
|
||||
# See if we are given a tzinfo object...
|
||||
tzinfo = None
|
||||
if isinstance(args[-1], (datetime.tzinfo, basestring)):
|
||||
tzinfo = _tzinfome(args.pop(-1))
|
||||
elif kw.get("tzinfo", None) is not None:
|
||||
tzinfo = _tzinfome(kw.pop("tzinfo"))
|
||||
|
||||
# Create a datetime object if we don't have one
|
||||
if isinstance(args[0], datetime.datetime):
|
||||
# Convert the datetime instance to a datetime object.
|
||||
newargs = (list(args[0].timetuple()[0:6]) +
|
||||
[args[0].microsecond, args[0].tzinfo])
|
||||
dt = datetime.datetime(*newargs)
|
||||
|
||||
if tzinfo is None and dt.tzinfo is None:
|
||||
raise TypeError("Must specify a timezone!")
|
||||
|
||||
if tzinfo is not None and dt.tzinfo is not None:
|
||||
raise TypeError("Can not give a timezone with timezone aware"
|
||||
" datetime object! (Use localize.)")
|
||||
else:
|
||||
dt = datetime.datetime(*args, **kw)
|
||||
|
||||
if dt.tzinfo is not None:
|
||||
# Re-normalize the dt object
|
||||
dt = dt.tzinfo.normalize(dt)
|
||||
|
||||
else:
|
||||
if tzinfo is None:
|
||||
tzinfo = localtz()
|
||||
|
||||
try:
|
||||
dt = tzinfo.localize(dt, is_dst=None)
|
||||
except pytz.AmbiguousTimeError:
|
||||
is_dst = None
|
||||
if "is_dst" in kw:
|
||||
is_dst = kw.pop("is_dst")
|
||||
|
||||
try:
|
||||
dt = tzinfo.localize(dt, is_dst)
|
||||
except IndexError:
|
||||
raise pytz.AmbiguousTimeError("No such time exists!")
|
||||
|
||||
newargs = list(dt.timetuple()[0:6])+[dt.microsecond, dt.tzinfo]
|
||||
obj = datetime.datetime.__new__(cls, *newargs)
|
||||
obj.is_dst = obj.dst() != datetime.timedelta(0)
|
||||
return obj
|
||||
|
||||
def asdatetime(self, naive=True):
|
||||
"""Return this datetime_tz as a datetime object.
|
||||
|
||||
Args:
|
||||
naive: Return *without* any tz info.
|
||||
|
||||
Returns:
|
||||
This datetime_tz as a datetime object.
|
||||
"""
|
||||
args = list(self.timetuple()[0:6])+[self.microsecond]
|
||||
if not naive:
|
||||
args.append(self.tzinfo)
|
||||
return datetime.datetime(*args)
|
||||
|
||||
def asdate(self):
|
||||
"""Return this datetime_tz as a date object.
|
||||
|
||||
Returns:
|
||||
This datetime_tz as a date object.
|
||||
"""
|
||||
return datetime.date(self.year, self.month, self.day)
|
||||
|
||||
def totimestamp(self):
|
||||
"""Convert this datetime object back to a unix timestamp.
|
||||
|
||||
The Unix epoch is the time 00:00:00 UTC on January 1, 1970.
|
||||
|
||||
Returns:
|
||||
Unix timestamp.
|
||||
"""
|
||||
return calendar.timegm(self.utctimetuple())+1e-6*self.microsecond
|
||||
|
||||
def astimezone(self, tzinfo):
|
||||
"""Returns a version of this timestamp converted to the given timezone.
|
||||
|
||||
Args:
|
||||
tzinfo: Either a datetime.tzinfo object or a string (which will be looked
|
||||
up in pytz.
|
||||
|
||||
Returns:
|
||||
A datetime_tz object in the given timezone.
|
||||
"""
|
||||
# Assert we are not a naive datetime object
|
||||
assert self.tzinfo is not None
|
||||
|
||||
tzinfo = _tzinfome(tzinfo)
|
||||
|
||||
d = self.asdatetime(naive=False).astimezone(tzinfo)
|
||||
return datetime_tz(d)
|
||||
|
||||
# pylint: disable-msg=C6113
|
||||
def replace(self, **kw):
|
||||
"""Return datetime with new specified fields given as arguments.
|
||||
|
||||
For example, dt.replace(days=4) would return a new datetime_tz object with
|
||||
exactly the same as dt but with the days attribute equal to 4.
|
||||
|
||||
Any attribute can be replaced, but tzinfo can not be set to None.
|
||||
|
||||
Args:
|
||||
Any datetime_tz attribute.
|
||||
|
||||
Returns:
|
||||
A datetime_tz object with the attributes replaced.
|
||||
|
||||
Raises:
|
||||
TypeError: If the given replacement is invalid.
|
||||
"""
|
||||
if "tzinfo" in kw:
|
||||
if kw["tzinfo"] is None:
|
||||
raise TypeError("Can not remove the timezone use asdatetime()")
|
||||
|
||||
is_dst = None
|
||||
if "is_dst" in kw:
|
||||
is_dst = kw["is_dst"]
|
||||
del kw["is_dst"]
|
||||
else:
|
||||
# Use our own DST setting..
|
||||
is_dst = self.is_dst
|
||||
|
||||
replaced = self.asdatetime().replace(**kw)
|
||||
|
||||
return datetime_tz(replaced, tzinfo=self.tzinfo.zone, is_dst=is_dst)
|
||||
|
||||
# pylint: disable-msg=C6310
|
||||
@classmethod
|
||||
def smartparse(cls, toparse, tzinfo=None):
|
||||
"""Method which uses dateutil.parse and extras to try and parse the string.
|
||||
|
||||
Valid dates are found at:
|
||||
http://labix.org/python-dateutil#head-1443e0f14ad5dff07efd465e080d1110920673d8-2
|
||||
|
||||
Other valid formats include:
|
||||
"now" or "today"
|
||||
"yesterday"
|
||||
"tommorrow"
|
||||
"5 minutes ago"
|
||||
"10 hours ago"
|
||||
"10h5m ago"
|
||||
"start of yesterday"
|
||||
"end of tommorrow"
|
||||
"end of 3rd of March"
|
||||
|
||||
Args:
|
||||
toparse: The string to parse.
|
||||
tzinfo: Timezone for the resultant datetime_tz object should be in.
|
||||
(Defaults to your local timezone.)
|
||||
|
||||
Returns:
|
||||
New datetime_tz object.
|
||||
|
||||
Raises:
|
||||
ValueError: If unable to make sense of the input.
|
||||
"""
|
||||
# Default for empty fields are:
|
||||
# year/month/day == now
|
||||
# hour/minute/second/microsecond == 0
|
||||
toparse = toparse.strip()
|
||||
|
||||
if tzinfo is None:
|
||||
dt = cls.now()
|
||||
else:
|
||||
dt = cls.now(tzinfo)
|
||||
|
||||
default = dt.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# Remove "start of " and "end of " prefix in the string
|
||||
if toparse.lower().startswith("end of "):
|
||||
toparse = toparse[7:].strip()
|
||||
|
||||
dt += datetime.timedelta(days=1)
|
||||
dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
dt -= datetime.timedelta(microseconds=1)
|
||||
|
||||
default = dt
|
||||
|
||||
elif toparse.lower().startswith("start of "):
|
||||
toparse = toparse[9:].strip()
|
||||
|
||||
dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
default = dt
|
||||
|
||||
# Handle strings with "now", "today", "yesterday", "tomorrow" and "ago".
|
||||
# Need to use lowercase
|
||||
toparselower = toparse.lower()
|
||||
|
||||
if toparselower in ["now", "today"]:
|
||||
pass
|
||||
|
||||
elif toparselower == "yesterday":
|
||||
dt -= datetime.timedelta(days=1)
|
||||
|
||||
elif toparselower == "tommorrow":
|
||||
dt += datetime.timedelta(days=1)
|
||||
|
||||
elif "ago" in toparselower:
|
||||
# Remove the "ago" bit
|
||||
toparselower = toparselower[:-3]
|
||||
# Replace all "a day and an hour" with "1 day 1 hour"
|
||||
toparselower = toparselower.replace("a ", "1 ")
|
||||
toparselower = toparselower.replace("an ", "1 ")
|
||||
toparselower = toparselower.replace(" and ", " ")
|
||||
|
||||
# Match the following
|
||||
# 1 hour ago
|
||||
# 1h ago
|
||||
# 1 h ago
|
||||
# 1 hour ago
|
||||
# 2 hours ago
|
||||
# Same with minutes, seconds, etc.
|
||||
|
||||
tocheck = ("seconds", "minutes", "hours", "days", "weeks", "months",
|
||||
"years")
|
||||
result = {}
|
||||
for match in re.finditer("([0-9]+)([^0-9]*)", toparselower):
|
||||
amount = int(match.group(1))
|
||||
unit = match.group(2).strip()
|
||||
|
||||
for bit in tocheck:
|
||||
regex = "^([%s]|((%s)s?))$" % (
|
||||
bit[0], bit[:-1])
|
||||
|
||||
bitmatch = re.search(regex, unit)
|
||||
if bitmatch:
|
||||
result[bit] = amount
|
||||
break
|
||||
else:
|
||||
raise ValueError("Was not able to parse date unit %r!" % unit)
|
||||
|
||||
delta = dateutil.relativedelta.relativedelta(**result)
|
||||
dt -= delta
|
||||
|
||||
else:
|
||||
# Handle strings with normal datetime format, use original case.
|
||||
dt = dateutil.parser.parse(toparse, default=default.asdatetime(),
|
||||
tzinfos=pytz_abbr.tzinfos)
|
||||
if dt is None:
|
||||
raise ValueError("Was not able to parse date!")
|
||||
|
||||
if dt.tzinfo is pytz_abbr.unknown:
|
||||
dt = dt.replace(tzinfo=None)
|
||||
|
||||
if dt.tzinfo is None:
|
||||
if tzinfo is None:
|
||||
tzinfo = localtz()
|
||||
dt = cls(dt, tzinfo)
|
||||
else:
|
||||
if isinstance(dt.tzinfo, pytz_abbr.tzabbr):
|
||||
abbr = dt.tzinfo
|
||||
dt = dt.replace(tzinfo=None)
|
||||
dt = cls(dt, abbr.zone, is_dst=abbr.dst)
|
||||
|
||||
dt = cls(dt)
|
||||
|
||||
return dt
|
||||
|
||||
@classmethod
|
||||
def utcfromtimestamp(cls, timestamp):
|
||||
"""Returns a datetime object of a given timestamp (in UTC)."""
|
||||
obj = datetime.datetime.utcfromtimestamp(timestamp)
|
||||
obj = pytz.utc.localize(obj)
|
||||
return cls(obj)
|
||||
|
||||
@classmethod
|
||||
def fromtimestamp(cls, timestamp):
|
||||
"""Returns a datetime object of a given timestamp (in local tz)."""
|
||||
d = cls.utcfromtimestamp(timestamp)
|
||||
return d.astimezone(localtz())
|
||||
|
||||
@classmethod
|
||||
def utcnow(cls):
|
||||
"""Return a new datetime representing UTC day and time."""
|
||||
obj = datetime.datetime.utcnow()
|
||||
obj = cls(obj, tzinfo=pytz.utc)
|
||||
return obj
|
||||
|
||||
@classmethod
|
||||
def now(cls, tzinfo=None):
|
||||
"""[tz] -> new datetime with tz's local day and time."""
|
||||
obj = cls.utcnow()
|
||||
if tzinfo is None:
|
||||
tzinfo = localtz()
|
||||
return obj.astimezone(tzinfo)
|
||||
|
||||
today = now
|
||||
|
||||
@staticmethod
|
||||
def fromordinal(ordinal):
|
||||
raise SyntaxError("Not enough information to create a datetime_tz object "
|
||||
"from an ordinal. Please use datetime.date.fromordinal")
|
||||
|
||||
|
||||
class iterate(object):
|
||||
"""Helpful iterators for working with datetime_tz objects."""
|
||||
|
||||
@staticmethod
|
||||
def between(start, delta, end=None):
|
||||
"""Return an iterator between this date till given end point.
|
||||
|
||||
Example usage:
|
||||
>>> d = datetime_tz.smartparse("5 days ago")
|
||||
2008/05/12 11:45
|
||||
>>> for i in d.between(timedelta(days=1), datetime_tz.now()):
|
||||
>>> print i
|
||||
2008/05/12 11:45
|
||||
2008/05/13 11:45
|
||||
2008/05/14 11:45
|
||||
2008/05/15 11:45
|
||||
2008/05/16 11:45
|
||||
|
||||
Args:
|
||||
start: The date to start at.
|
||||
delta: The interval to iterate with.
|
||||
end: (Optional) Date to end at. If not given the iterator will never
|
||||
terminate.
|
||||
|
||||
Yields:
|
||||
datetime_tz objects.
|
||||
"""
|
||||
toyield = start
|
||||
while end is None or toyield < end:
|
||||
yield toyield
|
||||
toyield += delta
|
||||
|
||||
@staticmethod
|
||||
def weeks(start, end=None):
|
||||
"""Iterate over the weeks between the given datetime_tzs.
|
||||
|
||||
Args:
|
||||
start: datetime_tz to start from.
|
||||
end: (Optional) Date to end at, if not given the iterator will never
|
||||
terminate.
|
||||
|
||||
Returns:
|
||||
An iterator which generates datetime_tz objects a week apart.
|
||||
"""
|
||||
return iterate.between(start, datetime.timedelta(days=7), end)
|
||||
|
||||
@staticmethod
|
||||
def days(start, end=None):
|
||||
"""Iterate over the days between the given datetime_tzs.
|
||||
|
||||
Args:
|
||||
start: datetime_tz to start from.
|
||||
end: (Optional) Date to end at, if not given the iterator will never
|
||||
terminate.
|
||||
|
||||
Returns:
|
||||
An iterator which generates datetime_tz objects a day apart.
|
||||
"""
|
||||
return iterate.between(start, datetime.timedelta(days=1), end)
|
||||
|
||||
@staticmethod
|
||||
def hours(start, end=None):
|
||||
"""Iterate over the hours between the given datetime_tzs.
|
||||
|
||||
Args:
|
||||
start: datetime_tz to start from.
|
||||
end: (Optional) Date to end at, if not given the iterator will never
|
||||
terminate.
|
||||
|
||||
Returns:
|
||||
An iterator which generates datetime_tz objects a hour apart.
|
||||
"""
|
||||
return iterate.between(start, datetime.timedelta(hours=1), end)
|
||||
|
||||
@staticmethod
|
||||
def minutes(start, end=None):
|
||||
"""Iterate over the minutes between the given datetime_tzs.
|
||||
|
||||
Args:
|
||||
start: datetime_tz to start from.
|
||||
end: (Optional) Date to end at, if not given the iterator will never
|
||||
terminate.
|
||||
|
||||
Returns:
|
||||
An iterator which generates datetime_tz objects a minute apart.
|
||||
"""
|
||||
return iterate.between(start, datetime.timedelta(minutes=1), end)
|
||||
|
||||
@staticmethod
|
||||
def seconds(start, end=None):
|
||||
"""Iterate over the seconds between the given datetime_tzs.
|
||||
|
||||
Args:
|
||||
start: datetime_tz to start from.
|
||||
end: (Optional) Date to end at, if not given the iterator will never
|
||||
terminate.
|
||||
|
||||
Returns:
|
||||
An iterator which generates datetime_tz objects a second apart.
|
||||
"""
|
||||
return iterate.between(start, datetime.timedelta(minutes=1), end)
|
||||
|
||||
|
||||
def _wrap_method(name):
|
||||
"""Wrap a method.
|
||||
|
||||
Patch a method which might return a datetime.datetime to return a
|
||||
datetime_tz.datetime_tz instead.
|
||||
|
||||
Args:
|
||||
name: The name of the method to patch
|
||||
"""
|
||||
method = getattr(datetime.datetime, name)
|
||||
|
||||
# Have to give the second argument as method has no __module__ option.
|
||||
@functools.wraps(method, ("__name__", "__doc__"), ())
|
||||
def wrapper(*args, **kw):
|
||||
r = method(*args, **kw)
|
||||
|
||||
if isinstance(r, datetime.datetime) and not isinstance(r, datetime_tz):
|
||||
r = datetime_tz(r)
|
||||
return r
|
||||
|
||||
setattr(datetime_tz, name, wrapper)
|
||||
|
||||
for methodname in ["__add__", "__radd__", "__rsub__", "__sub__", "combine"]:
|
||||
|
||||
# Make sure we have not already got an override for this method
|
||||
assert methodname not in datetime_tz.__dict__
|
||||
|
||||
_wrap_method(methodname)
|
||||
|
||||
|
||||
__all__ = ['datetime_tz', 'detect_timezone', 'iterate', 'localtz',
|
||||
'localtz_set', 'timedelta', '_detect_timezone_environ',
|
||||
'_detect_timezone_etc_localtime', '_detect_timezone_etc_timezone',
|
||||
'_detect_timezone_php']
|
||||
230
nilmdb/utils/datetime_tz/pytz_abbr.py
Normal file
230
nilmdb/utils/datetime_tz/pytz_abbr.py
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
#!/usr/bin/python2.4
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright 2010 Google Inc. All Rights Reserved.
|
||||
#
|
||||
|
||||
"""
|
||||
Common time zone acronyms/abbreviations for use with the datetime_tz module.
|
||||
|
||||
*WARNING*: There are lots of caveats when using this module which are listed
|
||||
below.
|
||||
|
||||
CAVEAT 1: The acronyms/abbreviations are not globally unique, they are not even
|
||||
unique within a region. For example, EST can mean any of,
|
||||
Eastern Standard Time in Australia (which is 10 hour ahead of UTC)
|
||||
Eastern Standard Time in North America (which is 5 hours behind UTC)
|
||||
|
||||
Where there are two abbreviations the more popular one will appear in the all
|
||||
dictionary, while the less common one will only appear in that countries region
|
||||
dictionary. IE If using all, EST will be mapped to Eastern Standard Time in
|
||||
North America.
|
||||
|
||||
CAVEAT 2: Many of the acronyms don't map to a neat Oslon timezones. For example,
|
||||
Eastern European Summer Time (EEDT) is used by many different countries in
|
||||
Europe *at different times*! If the acronym does not map neatly to one zone it
|
||||
is mapped to the Etc/GMT+-XX Oslon zone. This means that any date manipulations
|
||||
can end up with idiot things like summer time in the middle of winter.
|
||||
|
||||
CAVEAT 3: The Summer/Standard time difference is really important! For an hour
|
||||
each year it is needed to determine which time you are actually talking about.
|
||||
2002-10-27 01:20:00 EST != 2002-10-27 01:20:00 EDT
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import pytz
|
||||
import pytz.tzfile
|
||||
|
||||
|
||||
class tzabbr(datetime.tzinfo):
|
||||
"""A timezone abbreviation.
|
||||
|
||||
*WARNING*: This is not a tzinfo implementation! Trying to use this as tzinfo
|
||||
object will result in failure. We inherit from datetime.tzinfo so we can get
|
||||
through the dateutil checks.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
# A "marker" tzinfo object which is used to signify an unknown timezone.
|
||||
unknown = datetime.tzinfo(0)
|
||||
|
||||
|
||||
regions = {'all': {}, 'military': {}}
|
||||
# Create a special alias for the all and military regions
|
||||
all = regions['all']
|
||||
military = regions['military']
|
||||
|
||||
|
||||
def tzabbr_register(abbr, name, region, zone, dst):
|
||||
"""Register a new timezone abbreviation in the global registry.
|
||||
|
||||
If another abbreviation with the same name has already been registered it new
|
||||
abbreviation will only be registered in region specific dictionary.
|
||||
"""
|
||||
newabbr = tzabbr()
|
||||
newabbr.abbr = abbr
|
||||
newabbr.name = name
|
||||
newabbr.region = region
|
||||
newabbr.zone = zone
|
||||
newabbr.dst = dst
|
||||
|
||||
if abbr not in all:
|
||||
all[abbr] = newabbr
|
||||
|
||||
if not region in regions:
|
||||
regions[region] = {}
|
||||
|
||||
assert abbr not in regions[region]
|
||||
regions[region][abbr] = newabbr
|
||||
|
||||
|
||||
def tzinfos_create(use_region):
|
||||
abbrs = regions[use_region]
|
||||
|
||||
def tzinfos(abbr, offset):
|
||||
if abbr:
|
||||
if abbr in abbrs:
|
||||
result = abbrs[abbr]
|
||||
if offset:
|
||||
# FIXME: Check the offset matches the abbreviation we just selected.
|
||||
pass
|
||||
return result
|
||||
else:
|
||||
raise ValueError, "Unknown timezone found %s" % abbr
|
||||
if offset == 0:
|
||||
return pytz.utc
|
||||
if offset:
|
||||
return pytz.FixedOffset(offset/60)
|
||||
return unknown
|
||||
|
||||
return tzinfos
|
||||
|
||||
|
||||
# Create a special alias for the all tzinfos
|
||||
tzinfos = tzinfos_create('all')
|
||||
|
||||
|
||||
# Create the abbreviations.
|
||||
# *WARNING*: Order matters!
|
||||
tzabbr_register("A", u"Alpha Time Zone", u"Military", "Etc/GMT-1", False)
|
||||
tzabbr_register("ACDT", u"Australian Central Daylight Time", u"Australia",
|
||||
"Australia/Adelaide", True)
|
||||
tzabbr_register("ACST", u"Australian Central Standard Time", u"Australia",
|
||||
"Australia/Adelaide", False)
|
||||
tzabbr_register("ADT", u"Atlantic Daylight Time", u"North America",
|
||||
"America/Halifax", True)
|
||||
tzabbr_register("AEDT", u"Australian Eastern Daylight Time", u"Australia",
|
||||
"Australia/Sydney", True)
|
||||
tzabbr_register("AEST", u"Australian Eastern Standard Time", u"Australia",
|
||||
"Australia/Sydney", False)
|
||||
tzabbr_register("AKDT", u"Alaska Daylight Time", u"North America",
|
||||
"US/Alaska", True)
|
||||
tzabbr_register("AKST", u"Alaska Standard Time", u"North America",
|
||||
"US/Alaska", False)
|
||||
tzabbr_register("AST", u"Atlantic Standard Time", u"North America",
|
||||
"America/Halifax", False)
|
||||
tzabbr_register("AWDT", u"Australian Western Daylight Time", u"Australia",
|
||||
"Australia/West", True)
|
||||
tzabbr_register("AWST", u"Australian Western Standard Time", u"Australia",
|
||||
"Australia/West", False)
|
||||
tzabbr_register("B", u"Bravo Time Zone", u"Military", "Etc/GMT-2", False)
|
||||
tzabbr_register("BST", u"British Summer Time", u"Europe", "Europe/London", True)
|
||||
tzabbr_register("C", u"Charlie Time Zone", u"Military", "Etc/GMT-2", False)
|
||||
tzabbr_register("CDT", u"Central Daylight Time", u"North America",
|
||||
"US/Central", True)
|
||||
tzabbr_register("CEDT", u"Central European Daylight Time", u"Europe",
|
||||
"Etc/GMT+2", True)
|
||||
tzabbr_register("CEST", u"Central European Summer Time", u"Europe",
|
||||
"Etc/GMT+2", True)
|
||||
tzabbr_register("CET", u"Central European Time", u"Europe", "Etc/GMT+1", False)
|
||||
tzabbr_register("CST", u"Central Standard Time", u"North America",
|
||||
"US/Central", False)
|
||||
tzabbr_register("CXT", u"Christmas Island Time", u"Australia",
|
||||
"Indian/Christmas", False)
|
||||
tzabbr_register("D", u"Delta Time Zone", u"Military", "Etc/GMT-2", False)
|
||||
tzabbr_register("E", u"Echo Time Zone", u"Military", "Etc/GMT-2", False)
|
||||
tzabbr_register("EDT", u"Eastern Daylight Time", u"North America",
|
||||
"US/Eastern", True)
|
||||
tzabbr_register("EEDT", u"Eastern European Daylight Time", u"Europe",
|
||||
"Etc/GMT+3", True)
|
||||
tzabbr_register("EEST", u"Eastern European Summer Time", u"Europe",
|
||||
"Etc/GMT+3", True)
|
||||
tzabbr_register("EET", u"Eastern European Time", u"Europe", "Etc/GMT+2", False)
|
||||
tzabbr_register("EST", u"Eastern Standard Time", u"North America",
|
||||
"US/Eastern", False)
|
||||
tzabbr_register("F", u"Foxtrot Time Zone", u"Military", "Etc/GMT-6", False)
|
||||
tzabbr_register("G", u"Golf Time Zone", u"Military", "Etc/GMT-7", False)
|
||||
tzabbr_register("GMT", u"Greenwich Mean Time", u"Europe", pytz.utc, False)
|
||||
tzabbr_register("H", u"Hotel Time Zone", u"Military", "Etc/GMT-8", False)
|
||||
#tzabbr_register("HAA", u"Heure Avancée de l'Atlantique", u"North America", u"UTC - 3 hours")
|
||||
#tzabbr_register("HAC", u"Heure Avancée du Centre", u"North America", u"UTC - 5 hours")
|
||||
tzabbr_register("HADT", u"Hawaii-Aleutian Daylight Time", u"North America",
|
||||
"Pacific/Honolulu", True)
|
||||
#tzabbr_register("HAE", u"Heure Avancée de l'Est", u"North America", u"UTC - 4 hours")
|
||||
#tzabbr_register("HAP", u"Heure Avancée du Pacifique", u"North America", u"UTC - 7 hours")
|
||||
#tzabbr_register("HAR", u"Heure Avancée des Rocheuses", u"North America", u"UTC - 6 hours")
|
||||
tzabbr_register("HAST", u"Hawaii-Aleutian Standard Time", u"North America",
|
||||
"Pacific/Honolulu", False)
|
||||
#tzabbr_register("HAT", u"Heure Avancée de Terre-Neuve", u"North America", u"UTC - 2:30 hours")
|
||||
#tzabbr_register("HAY", u"Heure Avancée du Yukon", u"North America", u"UTC - 8 hours")
|
||||
tzabbr_register("HDT", u"Hawaii Daylight Time", u"North America",
|
||||
"Pacific/Honolulu", True)
|
||||
#tzabbr_register("HNA", u"Heure Normale de l'Atlantique", u"North America", u"UTC - 4 hours")
|
||||
#tzabbr_register("HNC", u"Heure Normale du Centre", u"North America", u"UTC - 6 hours")
|
||||
#tzabbr_register("HNE", u"Heure Normale de l'Est", u"North America", u"UTC - 5 hours")
|
||||
#tzabbr_register("HNP", u"Heure Normale du Pacifique", u"North America", u"UTC - 8 hours")
|
||||
#tzabbr_register("HNR", u"Heure Normale des Rocheuses", u"North America", u"UTC - 7 hours")
|
||||
#tzabbr_register("HNT", u"Heure Normale de Terre-Neuve", u"North America", u"UTC - 3:30 hours")
|
||||
#tzabbr_register("HNY", u"Heure Normale du Yukon", u"North America", u"UTC - 9 hours")
|
||||
tzabbr_register("HST", u"Hawaii Standard Time", u"North America",
|
||||
"Pacific/Honolulu", False)
|
||||
tzabbr_register("I", u"India Time Zone", u"Military", "Etc/GMT-9", False)
|
||||
tzabbr_register("IST", u"Irish Summer Time", u"Europe", "Europe/Dublin", True)
|
||||
tzabbr_register("K", u"Kilo Time Zone", u"Military", "Etc/GMT-10", False)
|
||||
tzabbr_register("L", u"Lima Time Zone", u"Military", "Etc/GMT-11", False)
|
||||
tzabbr_register("M", u"Mike Time Zone", u"Military", "Etc/GMT-12", False)
|
||||
tzabbr_register("MDT", u"Mountain Daylight Time", u"North America",
|
||||
"US/Mountain", True)
|
||||
#tzabbr_register("MESZ", u"Mitteleuroäische Sommerzeit", u"Europe", u"UTC + 2 hours")
|
||||
#tzabbr_register("MEZ", u"Mitteleuropäische Zeit", u"Europe", u"UTC + 1 hour")
|
||||
tzabbr_register("MSD", u"Moscow Daylight Time", u"Europe",
|
||||
"Europe/Moscow", True)
|
||||
tzabbr_register("MSK", u"Moscow Standard Time", u"Europe",
|
||||
"Europe/Moscow", False)
|
||||
tzabbr_register("MST", u"Mountain Standard Time", u"North America",
|
||||
"US/Mountain", False)
|
||||
tzabbr_register("N", u"November Time Zone", u"Military", "Etc/GMT+1", False)
|
||||
tzabbr_register("NDT", u"Newfoundland Daylight Time", u"North America",
|
||||
"America/St_Johns", True)
|
||||
tzabbr_register("NFT", u"Norfolk (Island) Time", u"Australia",
|
||||
"Pacific/Norfolk", False)
|
||||
tzabbr_register("NST", u"Newfoundland Standard Time", u"North America",
|
||||
"America/St_Johns", False)
|
||||
tzabbr_register("O", u"Oscar Time Zone", u"Military", "Etc/GMT+2", False)
|
||||
tzabbr_register("P", u"Papa Time Zone", u"Military", "Etc/GMT+3", False)
|
||||
tzabbr_register("PDT", u"Pacific Daylight Time", u"North America",
|
||||
"US/Pacific", True)
|
||||
tzabbr_register("PST", u"Pacific Standard Time", u"North America",
|
||||
"US/Pacific", False)
|
||||
tzabbr_register("Q", u"Quebec Time Zone", u"Military", "Etc/GMT+4", False)
|
||||
tzabbr_register("R", u"Romeo Time Zone", u"Military", "Etc/GMT+5", False)
|
||||
tzabbr_register("S", u"Sierra Time Zone", u"Military", "Etc/GMT+6", False)
|
||||
tzabbr_register("T", u"Tango Time Zone", u"Military", "Etc/GMT+7", False)
|
||||
tzabbr_register("U", u"Uniform Time Zone", u"Military", "Etc/GMT+8", False)
|
||||
tzabbr_register("UTC", u"Coordinated Universal Time", u"Europe",
|
||||
pytz.utc, False)
|
||||
tzabbr_register("V", u"Victor Time Zone", u"Military", "Etc/GMT+9", False)
|
||||
tzabbr_register("W", u"Whiskey Time Zone", u"Military", "Etc/GMT+10", False)
|
||||
tzabbr_register("WDT", u"Western Daylight Time", u"Australia",
|
||||
"Australia/West", True)
|
||||
tzabbr_register("WEDT", u"Western European Daylight Time", u"Europe",
|
||||
"Etc/GMT+1", True)
|
||||
tzabbr_register("WEST", u"Western European Summer Time", u"Europe",
|
||||
"Etc/GMT+1", True)
|
||||
tzabbr_register("WET", u"Western European Time", u"Europe", pytz.utc, False)
|
||||
tzabbr_register("WST", u"Western Standard Time", u"Australia",
|
||||
"Australia/West", False)
|
||||
tzabbr_register("X", u"X-ray Time Zone", u"Military", "Etc/GMT+11", False)
|
||||
tzabbr_register("Y", u"Yankee Time Zone", u"Military", "Etc/GMT+12", False)
|
||||
tzabbr_register("Z", u"Zulu Time Zone", u"Military", pytz.utc, False)
|
||||
|
|
@ -2,21 +2,19 @@ import os
|
|||
import errno
|
||||
from math import log
|
||||
|
||||
|
||||
def human_size(num):
|
||||
"""Human friendly file size"""
|
||||
unit_list = list(zip(['bytes', 'kiB', 'MiB', 'GiB', 'TiB'],
|
||||
[0, 0, 1, 2, 2]))
|
||||
if num == 0:
|
||||
return '0 bytes'
|
||||
if num == 1:
|
||||
return '1 byte'
|
||||
unit_list = zip(['bytes', 'kiB', 'MiB', 'GiB', 'TiB'], [0, 0, 1, 2, 2])
|
||||
if num > 1:
|
||||
exponent = min(int(log(num, 1024)), len(unit_list) - 1)
|
||||
quotient = float(num) / 1024**exponent
|
||||
unit, num_decimals = unit_list[exponent]
|
||||
format_string = '{:.%sf} {}' % (num_decimals)
|
||||
return format_string.format(quotient, unit)
|
||||
|
||||
if num == 0: # pragma: no cover
|
||||
return '0 bytes'
|
||||
if num == 1: # pragma: no cover
|
||||
return '1 byte'
|
||||
|
||||
def du(path):
|
||||
"""Like du -sb, returns total size of path in bytes. Ignore
|
||||
|
|
@ -30,7 +28,7 @@ def du(path):
|
|||
filepath = os.path.join(path, thisfile)
|
||||
size += du(filepath)
|
||||
return size
|
||||
except OSError as e:
|
||||
except OSError as e: # pragma: no cover
|
||||
if e.errno != errno.ENOENT:
|
||||
raise
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -1,20 +1,49 @@
|
|||
# Implementation of hole punching via fallocate, if the OS
|
||||
# and filesystem support it.
|
||||
|
||||
import fallocate
|
||||
try:
|
||||
import os
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
|
||||
def make_fallocate():
|
||||
libc_name = ctypes.util.find_library('c')
|
||||
libc = ctypes.CDLL(libc_name, use_errno=True)
|
||||
|
||||
def punch_hole(filename, offset, length, ignore_errors=True):
|
||||
_fallocate = libc.fallocate
|
||||
_fallocate.restype = ctypes.c_int
|
||||
_fallocate.argtypes = [ ctypes.c_int, ctypes.c_int,
|
||||
ctypes.c_int64, ctypes.c_int64 ]
|
||||
|
||||
del libc
|
||||
del libc_name
|
||||
|
||||
def fallocate(fd, mode, offset, len_):
|
||||
res = _fallocate(fd, mode, offset, len_)
|
||||
if res != 0: # pragma: no cover
|
||||
errno = ctypes.get_errno()
|
||||
raise IOError(errno, os.strerror(errno))
|
||||
return fallocate
|
||||
|
||||
fallocate = make_fallocate()
|
||||
del make_fallocate
|
||||
except Exception: # pragma: no cover
|
||||
fallocate = None
|
||||
|
||||
FALLOC_FL_KEEP_SIZE = 0x01
|
||||
FALLOC_FL_PUNCH_HOLE = 0x02
|
||||
|
||||
def punch_hole(filename, offset, length, ignore_errors = True):
|
||||
"""Punch a hole in the file. This isn't well supported, so errors
|
||||
are ignored by default."""
|
||||
try:
|
||||
if fallocate is None: # pragma: no cover
|
||||
raise IOError("fallocate not available")
|
||||
with open(filename, "r+") as f:
|
||||
fallocate.fallocate(
|
||||
f.fileno(),
|
||||
offset,
|
||||
length,
|
||||
fallocate.FALLOC_FL_KEEP_SIZE | fallocate.FALLOC_FL_PUNCH_HOLE)
|
||||
except Exception:
|
||||
fallocate(f.fileno(),
|
||||
FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
|
||||
offset, length)
|
||||
except IOError: # pragma: no cover
|
||||
if ignore_errors:
|
||||
return
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -9,12 +9,10 @@ Intervals are half-open, ie. they include data points with timestamps
|
|||
import nilmdb.utils.time
|
||||
import nilmdb.utils.iterator
|
||||
|
||||
|
||||
class IntervalError(Exception):
|
||||
"""Error due to interval overlap, etc"""
|
||||
pass
|
||||
|
||||
|
||||
# Interval
|
||||
class Interval:
|
||||
"""Represents an interval of time."""
|
||||
|
|
@ -24,7 +22,7 @@ class Interval:
|
|||
'start' and 'end' are arbitrary numbers that represent time
|
||||
"""
|
||||
if start >= end:
|
||||
# Explicitly disallow zero-width intervals, since they're half-open
|
||||
# Explicitly disallow zero-width intervals (since they're half-open)
|
||||
raise IntervalError("start %s must precede end %s" % (start, end))
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
|
@ -41,24 +39,9 @@ class Interval:
|
|||
return ("[ " + nilmdb.utils.time.timestamp_to_human(self.start) +
|
||||
" -> " + nilmdb.utils.time.timestamp_to_human(self.end) + " ]")
|
||||
|
||||
# Compare two intervals. If non-equal, order by start then end
|
||||
def __lt__(self, other):
|
||||
return (self.start, self.end) < (other.start, other.end)
|
||||
|
||||
def __gt__(self, other):
|
||||
return (self.start, self.end) > (other.start, other.end)
|
||||
|
||||
def __le__(self, other):
|
||||
return (self.start, self.end) <= (other.start, other.end)
|
||||
|
||||
def __ge__(self, other):
|
||||
return (self.start, self.end) >= (other.start, other.end)
|
||||
|
||||
def __eq__(self, other):
|
||||
return (self.start, self.end) == (other.start, other.end)
|
||||
|
||||
def __ne__(self, other):
|
||||
return (self.start, self.end) != (other.start, other.end)
|
||||
def __cmp__(self, other):
|
||||
"""Compare two intervals. If non-equal, order by start then end"""
|
||||
return cmp(self.start, other.start) or cmp(self.end, other.end)
|
||||
|
||||
def intersects(self, other):
|
||||
"""Return True if two Interval objects intersect"""
|
||||
|
|
@ -75,8 +58,7 @@ class Interval:
|
|||
raise IntervalError("not a subset")
|
||||
return Interval(start, end)
|
||||
|
||||
|
||||
def _interval_math_helper(a, b, op, subset=True):
|
||||
def _interval_math_helper(a, b, op, subset = True):
|
||||
"""Helper for set_difference, intersection functions,
|
||||
to compute interval subsets based on a math operator on ranges
|
||||
present in A and B. Subsets are computed from A, or new intervals
|
||||
|
|
@ -106,7 +88,7 @@ def _interval_math_helper(a, b, op, subset=True):
|
|||
in_b = True
|
||||
elif k == 2:
|
||||
in_a = False
|
||||
else: # k == 3
|
||||
elif k == 3:
|
||||
in_b = False
|
||||
include = op(in_a, in_b)
|
||||
if include and out_start is None:
|
||||
|
|
@ -119,7 +101,6 @@ def _interval_math_helper(a, b, op, subset=True):
|
|||
yield Interval(out_start, ts)
|
||||
out_start = None
|
||||
|
||||
|
||||
def set_difference(a, b):
|
||||
"""
|
||||
Compute the difference (a \\ b) between the intervals in 'a' and
|
||||
|
|
@ -134,7 +115,6 @@ def set_difference(a, b):
|
|||
"""
|
||||
return _interval_math_helper(a, b, (lambda a, b: a and not b))
|
||||
|
||||
|
||||
def intersection(a, b):
|
||||
"""
|
||||
Compute the intersection between the intervals in 'a' and the
|
||||
|
|
@ -149,7 +129,6 @@ def intersection(a, b):
|
|||
"""
|
||||
return _interval_math_helper(a, b, (lambda a, b: a and b))
|
||||
|
||||
|
||||
def optimize(it):
|
||||
"""
|
||||
Given an iterable 'it' with intervals, optimize them by joining
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@
|
|||
|
||||
# Iterator merging, based on http://code.activestate.com/recipes/491285/
|
||||
import heapq
|
||||
|
||||
|
||||
def imerge(*iterables):
|
||||
'''Merge multiple sorted inputs into a single sorted output.
|
||||
|
||||
|
|
@ -19,8 +17,8 @@ def imerge(*iterables):
|
|||
h_append = h.append
|
||||
for it in map(iter, iterables):
|
||||
try:
|
||||
nexter = it.__next__
|
||||
h_append([nexter(), nexter])
|
||||
next = it.next
|
||||
h_append([next(), next])
|
||||
except _Stop:
|
||||
pass
|
||||
heapq.heapify(h)
|
||||
|
|
@ -28,9 +26,9 @@ def imerge(*iterables):
|
|||
while 1:
|
||||
try:
|
||||
while 1:
|
||||
v, nexter = s = h[0] # raises IndexError when h is empty
|
||||
v, next = s = h[0] # raises IndexError when h is empty
|
||||
yield v
|
||||
s[0] = nexter() # raises StopIteration when exhausted
|
||||
s[0] = next() # raises StopIteration when exhausted
|
||||
siftup(h, 0) # restore heap condition
|
||||
except _Stop:
|
||||
heappop(h) # remove empty iterator
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# File locking
|
||||
|
||||
import fcntl
|
||||
import errno
|
||||
import warnings
|
||||
|
||||
try:
|
||||
import fcntl
|
||||
import errno
|
||||
|
||||
def exclusive_lock(f):
|
||||
def exclusive_lock(f):
|
||||
"""Acquire an exclusive lock. Returns True on successful
|
||||
lock, or False on error."""
|
||||
try:
|
||||
|
|
@ -12,11 +14,20 @@ def exclusive_lock(f):
|
|||
except IOError as e:
|
||||
if e.errno in (errno.EACCES, errno.EAGAIN):
|
||||
return False
|
||||
else:
|
||||
else: # pragma: no cover
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def exclusive_unlock(f):
|
||||
def exclusive_unlock(f):
|
||||
"""Release an exclusive lock."""
|
||||
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
except ImportError: # pragma: no cover
|
||||
def exclusive_lock(f):
|
||||
"""Dummy lock function -- does not lock!"""
|
||||
warnings.warn("Pretending to lock " + str(f))
|
||||
return True
|
||||
|
||||
def exclusive_unlock(f):
|
||||
"""Release an exclusive lock."""
|
||||
return
|
||||
|
|
|
|||
|
|
@ -6,11 +6,10 @@
|
|||
import collections
|
||||
import decorator
|
||||
|
||||
|
||||
def lru_cache(size=10, onremove=None, keys=slice(None)):
|
||||
def lru_cache(size = 10, onremove = None, keys = slice(None)):
|
||||
"""Least-recently-used cache decorator.
|
||||
|
||||
@lru_cache(size=10, onremove=None)
|
||||
@lru_cache(size = 10, onevict = None)
|
||||
def f(...):
|
||||
pass
|
||||
|
||||
|
|
@ -54,17 +53,14 @@ def lru_cache(size=10, onremove=None, keys=slice(None)):
|
|||
if key in cache:
|
||||
evict(cache.pop(key))
|
||||
else:
|
||||
if cache:
|
||||
if len(args) != len(next(iter(cache.keys()))):
|
||||
if len(cache) > 0 and len(args) != len(cache.iterkeys().next()):
|
||||
raise KeyError("trying to remove from LRU cache, but "
|
||||
"number of arguments doesn't match the "
|
||||
"cache key length")
|
||||
|
||||
def cache_remove_all():
|
||||
nonlocal cache
|
||||
for key in cache:
|
||||
evict(cache[key])
|
||||
cache = collections.OrderedDict()
|
||||
evict(cache.pop(key))
|
||||
|
||||
def cache_info():
|
||||
return (func.cache_hits, func.cache_misses)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
from nilmdb.utils.printf import *
|
||||
import sys
|
||||
import inspect
|
||||
import decorator
|
||||
from nilmdb.utils.printf import fprintf
|
||||
|
||||
|
||||
def must_close(errorfile=sys.stderr, wrap_verify=False):
|
||||
def must_close(errorfile = sys.stderr, wrap_verify = False):
|
||||
"""Class decorator that warns on 'errorfile' at deletion time if
|
||||
the class's close() member wasn't called.
|
||||
|
||||
|
|
@ -13,17 +12,12 @@ def must_close(errorfile=sys.stderr, wrap_verify=False):
|
|||
already been called."""
|
||||
def class_decorator(cls):
|
||||
|
||||
def is_method_or_function(x):
|
||||
return inspect.ismethod(x) or inspect.isfunction(x)
|
||||
|
||||
def wrap_class_method(wrapper):
|
||||
try:
|
||||
orig = getattr(cls, wrapper.__name__)
|
||||
except AttributeError:
|
||||
orig = getattr(cls, wrapper.__name__).im_func
|
||||
except Exception:
|
||||
orig = lambda x: None
|
||||
if is_method_or_function(orig):
|
||||
setattr(cls, wrapper.__name__,
|
||||
decorator.decorator(wrapper, orig))
|
||||
setattr(cls, wrapper.__name__, decorator.decorator(wrapper, orig))
|
||||
|
||||
@wrap_class_method
|
||||
def __init__(orig, self, *args, **kwargs):
|
||||
|
|
@ -39,7 +33,7 @@ def must_close(errorfile=sys.stderr, wrap_verify=False):
|
|||
fprintf(errorfile, "error: %s.close() wasn't called!\n",
|
||||
self.__class__.__name__)
|
||||
return orig(self, *args, **kwargs)
|
||||
except:
|
||||
except: # pragma: no cover
|
||||
pass
|
||||
|
||||
@wrap_class_method
|
||||
|
|
@ -55,17 +49,16 @@ def must_close(errorfile=sys.stderr, wrap_verify=False):
|
|||
raise AssertionError("called " + str(orig) + " after close")
|
||||
return orig(self, *args, **kwargs)
|
||||
if wrap_verify:
|
||||
for (name, method) in inspect.getmembers(cls,
|
||||
is_method_or_function):
|
||||
for (name, method) in inspect.getmembers(cls, inspect.ismethod):
|
||||
# Skip class methods
|
||||
if method.__self__ is not None:
|
||||
continue
|
||||
# Skip some methods
|
||||
if name in ["__del__", "__init__"]:
|
||||
if name in [ "__del__", "__init__" ]:
|
||||
continue
|
||||
# Set up wrapper
|
||||
if inspect.ismethod(method):
|
||||
func = method.__func__
|
||||
else:
|
||||
func = method
|
||||
setattr(cls, name, decorator.decorator(verifier, func))
|
||||
setattr(cls, name, decorator.decorator(verifier,
|
||||
method.im_func))
|
||||
|
||||
return cls
|
||||
return class_decorator
|
||||
|
|
|
|||
|
|
@ -1,13 +1,9 @@
|
|||
"""printf, fprintf, sprintf"""
|
||||
|
||||
|
||||
from __future__ import print_function
|
||||
def printf(_str, *args):
|
||||
print(_str % args, end='')
|
||||
|
||||
|
||||
def fprintf(_file, _str, *args):
|
||||
print(_str % args, end='', file=_file)
|
||||
|
||||
|
||||
def sprintf(_str, *args):
|
||||
return (_str % args)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import queue
|
||||
import Queue
|
||||
import threading
|
||||
import sys
|
||||
import decorator
|
||||
import inspect
|
||||
import types
|
||||
import functools
|
||||
|
||||
# This file provides a class that will wrap an object and serialize
|
||||
# all calls to its methods. All calls to that object will be queued
|
||||
|
|
@ -9,7 +13,6 @@ import sys
|
|||
|
||||
# Based partially on http://stackoverflow.com/questions/2642515/
|
||||
|
||||
|
||||
class SerializerThread(threading.Thread):
|
||||
"""Thread that retrieves call information from the queue, makes the
|
||||
call, and returns the results."""
|
||||
|
|
@ -37,7 +40,6 @@ class SerializerThread(threading.Thread):
|
|||
result_queue.put((exception, result))
|
||||
del exception, result
|
||||
|
||||
|
||||
def serializer_proxy(obj_or_type):
|
||||
"""Wrap the given object or type in a SerializerObjectProxy.
|
||||
|
||||
|
|
@ -47,54 +49,45 @@ def serializer_proxy(obj_or_type):
|
|||
The proxied requests, including instantiation, are performed in a
|
||||
single thread and serialized between caller threads.
|
||||
"""
|
||||
class SerializerCallProxy():
|
||||
class SerializerCallProxy(object):
|
||||
def __init__(self, call_queue, func, objectproxy):
|
||||
self.call_queue = call_queue
|
||||
self.func = func
|
||||
# Need to hold a reference to object proxy so it doesn't
|
||||
# go away (and kill the thread) until after get called.
|
||||
self.objectproxy = objectproxy
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
result_queue = queue.Queue()
|
||||
result_queue = Queue.Queue()
|
||||
self.call_queue.put((result_queue, self.func, args, kwargs))
|
||||
(exc_info, result) = result_queue.get()
|
||||
( exc_info, result ) = result_queue.get()
|
||||
if exc_info is None:
|
||||
return result
|
||||
else:
|
||||
raise exc_info[1].with_traceback(exc_info[2])
|
||||
raise exc_info[0], exc_info[1], exc_info[2]
|
||||
|
||||
class SerializerObjectProxy():
|
||||
class SerializerObjectProxy(object):
|
||||
def __init__(self, obj_or_type, *args, **kwargs):
|
||||
self.__object = obj_or_type
|
||||
if isinstance(obj_or_type, type):
|
||||
try:
|
||||
if type(obj_or_type) in (types.TypeType, types.ClassType):
|
||||
classname = obj_or_type.__name__
|
||||
else:
|
||||
classname = obj_or_type.__class__.__name__
|
||||
self.__call_queue = queue.Queue()
|
||||
except AttributeError: # pragma: no cover
|
||||
classname = "???"
|
||||
self.__call_queue = Queue.Queue()
|
||||
self.__thread = SerializerThread(classname, self.__call_queue)
|
||||
self.__thread.daemon = True
|
||||
self.__thread.start()
|
||||
self._thread_safe = True
|
||||
|
||||
def __getattr__(self, key):
|
||||
# If the attribute is a function, we want to return a
|
||||
# proxy that will perform the call through the serializer
|
||||
# when called. Otherwise, we want to return the value
|
||||
# directly. This means we need to grab the attribute once,
|
||||
# and therefore self.__object.__getattr__ may be called
|
||||
# in an unsafe way, from the caller's thread.
|
||||
if key.startswith("_SerializerObjectProxy__"): # pragma: no cover
|
||||
raise AttributeError
|
||||
attr = getattr(self.__object, key)
|
||||
if not callable(attr):
|
||||
# It's not callable, so perform the getattr from within
|
||||
# the serializer thread, then return its value.
|
||||
# That may differ from the "attr" value we just grabbed
|
||||
# from here, due to forced ordering in the serializer.
|
||||
getter = SerializerCallProxy(self.__call_queue, getattr, self)
|
||||
return getter(self.__object, key)
|
||||
else:
|
||||
# It is callable, so return an object that will proxy through
|
||||
# the serializer when called.
|
||||
r = SerializerCallProxy(self.__call_queue, attr, self)
|
||||
return r
|
||||
|
||||
|
|
@ -105,10 +98,9 @@ def serializer_proxy(obj_or_type):
|
|||
attr = getattr(self.__object, "__iter__")
|
||||
self.__iter = SerializerCallProxy(self.__call_queue, attr, self)()
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
def next(self):
|
||||
return SerializerCallProxy(self.__call_queue,
|
||||
self.__iter.__next__, self)()
|
||||
self.__iter.next, self)()
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.__getattr__("__getitem__")(key)
|
||||
|
|
@ -118,7 +110,7 @@ def serializer_proxy(obj_or_type):
|
|||
to serializer_proxy. Otherwise, pass the call through."""
|
||||
ret = SerializerCallProxy(self.__call_queue,
|
||||
self.__object, self)(*args, **kwargs)
|
||||
if isinstance(self.__object, type):
|
||||
if type(self.__object) in (types.TypeType, types.ClassType):
|
||||
# Instantiation
|
||||
self.__object = ret
|
||||
return self
|
||||
|
|
@ -126,9 +118,9 @@ def serializer_proxy(obj_or_type):
|
|||
|
||||
def __del__(self):
|
||||
try:
|
||||
# Signal thread to exit, but don't wait for it.
|
||||
self.__call_queue.put((None, None, None, None))
|
||||
except:
|
||||
self.__thread.join()
|
||||
except: # pragma: no cover
|
||||
pass
|
||||
|
||||
return SerializerObjectProxy(obj_or_type)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import re
|
||||
|
||||
|
||||
def sort_human(items, key=None):
|
||||
def sort_human(items, key = None):
|
||||
"""Human-friendly sort (/stream/2 before /stream/10)"""
|
||||
def to_num(val):
|
||||
try:
|
||||
|
|
@ -14,6 +13,6 @@ def sort_human(items, key=None):
|
|||
text = key(text)
|
||||
# Break into character and numeric chunks.
|
||||
chunks = re.split(r'([0-9]+)', text)
|
||||
return [to_num(c) for c in chunks]
|
||||
return [ to_num(c) for c in chunks ]
|
||||
|
||||
return sorted(items, key=human_key)
|
||||
return sorted(items, key = human_key)
|
||||
|
|
|
|||
|
|
@ -1,25 +1,26 @@
|
|||
from nilmdb.utils.printf import *
|
||||
import threading
|
||||
from nilmdb.utils.printf import sprintf
|
||||
import warnings
|
||||
import types
|
||||
|
||||
|
||||
def verify_proxy(obj_or_type, check_thread=True,
|
||||
check_concurrent=True):
|
||||
def verify_proxy(obj_or_type, exception = False, check_thread = True,
|
||||
check_concurrent = True):
|
||||
"""Wrap the given object or type in a VerifyObjectProxy.
|
||||
|
||||
Returns a VerifyObjectProxy that proxies all method calls to the
|
||||
given object, as well as attribute retrievals.
|
||||
|
||||
When calling methods, the following checks are performed. On
|
||||
failure, an exception is raised.
|
||||
When calling methods, the following checks are performed. If
|
||||
exception is True, an exception is raised. Otherwise, a warning
|
||||
is printed.
|
||||
|
||||
check_thread = True # Fail if two different threads call methods.
|
||||
check_concurrent = True # Fail if two functions are concurrently
|
||||
check_thread = True # Warn/fail if two different threads call methods.
|
||||
check_concurrent = True # Warn/fail if two functions are concurrently
|
||||
# run through this proxy
|
||||
"""
|
||||
class Namespace():
|
||||
class Namespace(object):
|
||||
pass
|
||||
|
||||
class VerifyCallProxy():
|
||||
class VerifyCallProxy(object):
|
||||
def __init__(self, func, parent_namespace):
|
||||
self.func = func
|
||||
self.parent_namespace = parent_namespace
|
||||
|
|
@ -41,16 +42,22 @@ def verify_proxy(obj_or_type, check_thread=True,
|
|||
" but %s called %s.%s",
|
||||
p.thread.name, p.classname, p.thread_callee,
|
||||
this.name, p.classname, callee)
|
||||
if exception:
|
||||
raise AssertionError(err)
|
||||
else: # pragma: no cover
|
||||
warnings.warn(err)
|
||||
|
||||
need_concur_unlock = False
|
||||
if check_concurrent:
|
||||
if not p.concur_lock.acquire(False):
|
||||
if p.concur_lock.acquire(False) == False:
|
||||
err = sprintf("unsafe concurrency: %s called %s.%s "
|
||||
"while %s is still in %s.%s",
|
||||
this.name, p.classname, callee,
|
||||
p.concur_tname, p.classname, p.concur_callee)
|
||||
if exception:
|
||||
raise AssertionError(err)
|
||||
else: # pragma: no cover
|
||||
warnings.warn(err)
|
||||
else:
|
||||
p.concur_tname = this.name
|
||||
p.concur_callee = callee
|
||||
|
|
@ -63,7 +70,7 @@ def verify_proxy(obj_or_type, check_thread=True,
|
|||
p.concur_lock.release()
|
||||
return ret
|
||||
|
||||
class VerifyObjectProxy():
|
||||
class VerifyObjectProxy(object):
|
||||
def __init__(self, obj_or_type, *args, **kwargs):
|
||||
p = Namespace()
|
||||
self.__ns = p
|
||||
|
|
@ -73,12 +80,17 @@ def verify_proxy(obj_or_type, check_thread=True,
|
|||
p.concur_tname = None
|
||||
p.concur_callee = None
|
||||
self.__obj = obj_or_type
|
||||
if isinstance(obj_or_type, type):
|
||||
try:
|
||||
if type(obj_or_type) in (types.TypeType, types.ClassType):
|
||||
p.classname = self.__obj.__name__
|
||||
else:
|
||||
p.classname = self.__obj.__class__.__name__
|
||||
except AttributeError: # pragma: no cover
|
||||
p.classname = "???"
|
||||
|
||||
def __getattr__(self, key):
|
||||
if key.startswith("_VerifyObjectProxy__"): # pragma: no cover
|
||||
raise AttributeError
|
||||
attr = getattr(self.__obj, key)
|
||||
if not callable(attr):
|
||||
return VerifyCallProxy(getattr, self.__ns)(self.__obj, key)
|
||||
|
|
@ -88,7 +100,7 @@ def verify_proxy(obj_or_type, check_thread=True,
|
|||
"""Call this to instantiate the type, if a type was passed
|
||||
to verify_proxy. Otherwise, pass the call through."""
|
||||
ret = VerifyCallProxy(self.__obj, self.__ns)(*args, **kwargs)
|
||||
if isinstance(self.__obj, type):
|
||||
if type(self.__obj) in (types.TypeType, types.ClassType):
|
||||
# Instantiation
|
||||
self.__obj = ret
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from nilmdb.utils import datetime_tz
|
||||
import re
|
||||
import time
|
||||
import datetime_tz
|
||||
|
||||
# Range
|
||||
min_timestamp = (-2**63)
|
||||
|
|
@ -9,17 +11,15 @@ max_timestamp = (2**63 - 1)
|
|||
# Smallest representable step
|
||||
epsilon = 1
|
||||
|
||||
|
||||
def string_to_timestamp(string):
|
||||
def string_to_timestamp(str):
|
||||
"""Convert a string that represents an integer number of microseconds
|
||||
since epoch."""
|
||||
try:
|
||||
# Parse a string like "1234567890123456" and return an integer
|
||||
return int(string)
|
||||
return int(str)
|
||||
except ValueError:
|
||||
# Try parsing as a float, in case it's "1234567890123456.0"
|
||||
return int(round(float(string)))
|
||||
|
||||
return int(round(float(str)))
|
||||
|
||||
def timestamp_to_string(timestamp):
|
||||
"""Convert a timestamp (integer microseconds since epoch) to a string"""
|
||||
|
|
@ -28,13 +28,6 @@ def timestamp_to_string(timestamp):
|
|||
else:
|
||||
return str(timestamp)
|
||||
|
||||
|
||||
def timestamp_to_bytes(timestamp):
|
||||
"""Convert a timestamp (integer microseconds since epoch) to a Python
|
||||
bytes object"""
|
||||
return timestamp_to_string(timestamp).encode('utf-8')
|
||||
|
||||
|
||||
def timestamp_to_human(timestamp):
|
||||
"""Convert a timestamp (integer microseconds since epoch) to a
|
||||
human-readable string, using the local timezone for display
|
||||
|
|
@ -46,30 +39,24 @@ def timestamp_to_human(timestamp):
|
|||
dt = datetime_tz.datetime_tz.fromtimestamp(timestamp_to_unix(timestamp))
|
||||
return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
|
||||
|
||||
|
||||
def unix_to_timestamp(unix):
|
||||
"""Convert a Unix timestamp (floating point seconds since epoch)
|
||||
into a NILM timestamp (integer microseconds since epoch)"""
|
||||
return int(round(unix * 1e6))
|
||||
|
||||
seconds_to_timestamp = unix_to_timestamp
|
||||
|
||||
def timestamp_to_unix(timestamp):
|
||||
"""Convert a NILM timestamp (integer microseconds since epoch)
|
||||
into a Unix timestamp (floating point seconds since epoch)"""
|
||||
return timestamp / 1e6
|
||||
|
||||
|
||||
seconds_to_timestamp = unix_to_timestamp
|
||||
timestamp_to_seconds = timestamp_to_unix
|
||||
|
||||
|
||||
def rate_to_period(hz, cycles=1):
|
||||
def rate_to_period(hz, cycles = 1):
|
||||
"""Convert a rate (in Hz) to a period (in timestamp units).
|
||||
Returns an integer."""
|
||||
period = unix_to_timestamp(cycles) / float(hz)
|
||||
return int(round(period))
|
||||
|
||||
|
||||
def parse_time(toparse):
|
||||
"""
|
||||
Parse a free-form time string and return a nilmdb timestamp
|
||||
|
|
@ -108,9 +95,9 @@ def parse_time(toparse):
|
|||
try:
|
||||
val = float(toparse)
|
||||
# range is from about year 2001 - 2128
|
||||
if 1e9 < val < 5e9:
|
||||
if val > 1e9 and val < 5e9:
|
||||
return unix_to_timestamp(val)
|
||||
if 1e15 < val < 5e15:
|
||||
if val > 1e15 and val < 5e15:
|
||||
return val
|
||||
except ValueError:
|
||||
pass
|
||||
|
|
@ -142,7 +129,6 @@ def parse_time(toparse):
|
|||
# just give up for now.
|
||||
raise ValueError("unable to parse timestamp")
|
||||
|
||||
|
||||
def now():
|
||||
"""Return current timestamp"""
|
||||
return unix_to_timestamp(time.time())
|
||||
|
|
|
|||
|
|
@ -5,17 +5,18 @@
|
|||
# with nilmdb.utils.Timer("flush"):
|
||||
# foo.flush()
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
import contextlib
|
||||
import time
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def Timer(name=None, tosyslog=False):
|
||||
def Timer(name = None, tosyslog = False):
|
||||
start = time.time()
|
||||
yield
|
||||
elapsed = int((time.time() - start) * 1000)
|
||||
msg = (name or 'elapsed') + ": " + str(elapsed) + " ms"
|
||||
if tosyslog:
|
||||
if tosyslog: # pragma: no cover
|
||||
import syslog
|
||||
syslog.syslog(msg)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
"""File-like objects that add timestamps to the input lines"""
|
||||
|
||||
from nilmdb.utils.printf import sprintf
|
||||
from nilmdb.utils.printf import *
|
||||
import nilmdb.utils.time
|
||||
|
||||
|
||||
class Timestamper():
|
||||
class Timestamper(object):
|
||||
"""A file-like object that adds timestamps to lines of an input file."""
|
||||
def __init__(self, infile, ts_iter):
|
||||
"""file: filename, or another file-like object
|
||||
ts_iter: iterator that returns a timestamp string for
|
||||
each line of the file"""
|
||||
if isinstance(infile, str):
|
||||
self.file = open(infile, "rb")
|
||||
if isinstance(infile, basestring):
|
||||
self.file = open(infile, "r")
|
||||
else:
|
||||
self.file = infile
|
||||
self.ts_iter = ts_iter
|
||||
|
|
@ -23,19 +22,17 @@ class Timestamper():
|
|||
while True:
|
||||
line = self.file.readline(*args)
|
||||
if not line:
|
||||
return b""
|
||||
if line[0:1] == b'#':
|
||||
return ""
|
||||
if line[0] == '#':
|
||||
continue
|
||||
# For some reason, coverage on python 3.8 reports that
|
||||
# we never hit this break, even though we definitely do.
|
||||
break # pragma: no cover
|
||||
break
|
||||
try:
|
||||
return next(self.ts_iter) + line
|
||||
return self.ts_iter.next() + line
|
||||
except StopIteration:
|
||||
return b""
|
||||
return ""
|
||||
|
||||
def readlines(self, size=None):
|
||||
out = b""
|
||||
def readlines(self, size = None):
|
||||
out = ""
|
||||
while True:
|
||||
line = self.readline()
|
||||
out += line
|
||||
|
|
@ -46,16 +43,15 @@ class Timestamper():
|
|||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
def next(self):
|
||||
result = self.readline()
|
||||
if not result:
|
||||
raise StopIteration
|
||||
return result
|
||||
|
||||
|
||||
class TimestamperRate(Timestamper):
|
||||
"""Timestamper that uses a start time and a fixed rate"""
|
||||
def __init__(self, infile, start, rate, end=None):
|
||||
def __init__(self, infile, start, rate, end = None):
|
||||
"""
|
||||
file: file name or object
|
||||
|
||||
|
|
@ -65,39 +61,33 @@ class TimestamperRate(Timestamper):
|
|||
|
||||
end: If specified, raise StopIteration before outputting a value
|
||||
greater than this."""
|
||||
timestamp_to_bytes = nilmdb.utils.time.timestamp_to_bytes
|
||||
timestamp_to_string = nilmdb.utils.time.timestamp_to_string
|
||||
rate_to_period = nilmdb.utils.time.rate_to_period
|
||||
|
||||
def iterator(start, rate, end):
|
||||
n = 0
|
||||
rate = float(rate)
|
||||
while True:
|
||||
now = start + rate_to_period(rate, n)
|
||||
if end and now >= end:
|
||||
return
|
||||
yield timestamp_to_bytes(now) + b" "
|
||||
raise StopIteration
|
||||
yield timestamp_to_string(now) + " "
|
||||
n += 1
|
||||
Timestamper.__init__(self, infile, iterator(start, rate, end))
|
||||
self.start = start
|
||||
self.rate = rate
|
||||
|
||||
def __str__(self):
|
||||
return sprintf("TimestamperRate(..., start=\"%s\", rate=%g)",
|
||||
nilmdb.utils.time.timestamp_to_human(self.start),
|
||||
self.rate)
|
||||
|
||||
|
||||
class TimestamperNow(Timestamper):
|
||||
"""Timestamper that uses current time"""
|
||||
def __init__(self, infile):
|
||||
timestamp_to_bytes = nilmdb.utils.time.timestamp_to_bytes
|
||||
timestamp_to_string = nilmdb.utils.time.timestamp_to_string
|
||||
get_now = nilmdb.utils.time.now
|
||||
|
||||
def iterator():
|
||||
while True:
|
||||
yield timestamp_to_bytes(get_now()) + b" "
|
||||
|
||||
yield timestamp_to_string(get_now()) + " "
|
||||
Timestamper.__init__(self, infile, iterator())
|
||||
|
||||
def __str__(self):
|
||||
return "TimestamperNow(...)"
|
||||
|
|
|
|||
29
nilmdb/utils/unicode.py
Normal file
29
nilmdb/utils/unicode.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import sys
|
||||
|
||||
if sys.version_info[0] >= 3: # pragma: no cover (future Python3 compat)
|
||||
text_type = str
|
||||
else:
|
||||
text_type = unicode
|
||||
|
||||
def encode(u):
|
||||
"""Try to encode something from Unicode to a string using the
|
||||
default encoding. If it fails, try encoding as UTF-8."""
|
||||
if not isinstance(u, text_type):
|
||||
return u
|
||||
try:
|
||||
return u.encode()
|
||||
except UnicodeEncodeError:
|
||||
return u.encode("utf-8")
|
||||
|
||||
def decode(s):
|
||||
"""Try to decode someting from string to Unicode using the
|
||||
default encoding. If it fails, try decoding as UTF-8."""
|
||||
if isinstance(s, text_type):
|
||||
return s
|
||||
try:
|
||||
return s.decode()
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
return s.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return s # best we can do
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
argcomplete==1.12.0
|
||||
CherryPy==18.6.0
|
||||
coverage==5.2.1
|
||||
Cython==0.29.21
|
||||
decorator==4.4.2
|
||||
fallocate==1.6.4
|
||||
flake8==3.8.3
|
||||
nose==1.3.7
|
||||
numpy==1.19.1
|
||||
progressbar==2.5
|
||||
psutil==5.7.2
|
||||
python-datetime-tz==0.5.4
|
||||
python-dateutil==2.8.1
|
||||
requests==2.24.0
|
||||
tz==0.2.2
|
||||
yappi==1.2.5
|
||||
|
||||
## The following requirements were added by pip freeze:
|
||||
beautifulsoup4==4.9.1
|
||||
certifi==2020.6.20
|
||||
chardet==3.0.4
|
||||
cheroot==8.4.2
|
||||
idna==2.10
|
||||
jaraco.classes==3.1.0
|
||||
jaraco.collections==3.0.0
|
||||
jaraco.functools==3.0.1
|
||||
jaraco.text==3.2.0
|
||||
mccabe==0.6.1
|
||||
more-itertools==8.4.0
|
||||
portend==2.6
|
||||
pycodestyle==2.6.0
|
||||
pyflakes==2.2.0
|
||||
pytz==2020.1
|
||||
six==1.15.0
|
||||
soupsieve==2.0.1
|
||||
tempora==4.0.0
|
||||
urllib3==1.25.10
|
||||
waitress==1.4.4
|
||||
WebOb==1.8.6
|
||||
WebTest==2.0.35
|
||||
zc.lockfile==2.0
|
||||
22
setup.cfg
22
setup.cfg
|
|
@ -13,6 +13,8 @@ cover-package=nilmdb
|
|||
cover-erase=1
|
||||
# this works, puts html output in cover/ dir:
|
||||
# cover-html=1
|
||||
# need nose 1.1.3 for this:
|
||||
# cover-branches=1
|
||||
#debug=nose
|
||||
#debug-log=nose.log
|
||||
stop=1
|
||||
|
|
@ -37,23 +39,3 @@ tests=tests
|
|||
#with-profile=1
|
||||
#profile-sort=time
|
||||
##profile-restrict=10 # doesn't work right, treated as string or something
|
||||
|
||||
[versioneer]
|
||||
VCS=git
|
||||
style=pep440
|
||||
versionfile_source=nilmdb/_version.py
|
||||
versionfile_build=nilmdb/_version.py
|
||||
tag_prefix=nilmdb-
|
||||
parentdir_prefix=nilmdb-
|
||||
|
||||
[flake8]
|
||||
exclude=_version.py
|
||||
extend-ignore=E731
|
||||
per-file-ignores=__init__.py:F401,E402 \
|
||||
serializer.py:E722 \
|
||||
mustclose.py:E722 \
|
||||
fsck.py:E266
|
||||
|
||||
[pylint]
|
||||
ignore=_version.py
|
||||
disable=C0103,C0111,R0913,R0914
|
||||
|
|
|
|||
104
setup.py
104
setup.py
|
|
@ -1,51 +1,119 @@
|
|||
#!/usr/bin/env python3
|
||||
#!/usr/bin/python
|
||||
|
||||
# To release a new version, tag it:
|
||||
# git tag -a nilmdb-1.1 -m "Version 1.1"
|
||||
# git push --tags
|
||||
# Then just package it up:
|
||||
# python3 setup.py sdist
|
||||
# python setup.py sdist
|
||||
|
||||
import traceback
|
||||
import sys
|
||||
import os
|
||||
from setuptools import setup
|
||||
from distutils.extension import Extension
|
||||
|
||||
try:
|
||||
from setuptools import setup, find_packages
|
||||
from distutils.extension import Extension
|
||||
import distutils.version
|
||||
except ImportError:
|
||||
traceback.print_exc()
|
||||
print "Please install the prerequisites listed in README.txt"
|
||||
sys.exit(1)
|
||||
|
||||
# Versioneer manages version numbers from git tags.
|
||||
# https://github.com/warner/python-versioneer
|
||||
import versioneer
|
||||
versioneer.versionfile_source = 'nilmdb/_version.py'
|
||||
versioneer.versionfile_build = 'nilmdb/_version.py'
|
||||
versioneer.tag_prefix = 'nilmdb-'
|
||||
versioneer.parentdir_prefix = 'nilmdb-'
|
||||
|
||||
# Hack to workaround logging/multiprocessing issue:
|
||||
# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
|
||||
try: import multiprocessing
|
||||
except Exception: pass
|
||||
|
||||
# Use Cython if it's new enough, otherwise use preexisting C files.
|
||||
cython_modules = [ 'nilmdb.server.interval',
|
||||
'nilmdb.server.rbtree' ]
|
||||
try:
|
||||
import Cython
|
||||
from Cython.Build import cythonize
|
||||
if (distutils.version.LooseVersion(Cython.__version__) <
|
||||
distutils.version.LooseVersion("0.16")):
|
||||
print "Cython version", Cython.__version__, "is too old; not using it."
|
||||
raise ImportError()
|
||||
use_cython = True
|
||||
except ImportError:
|
||||
use_cython = False
|
||||
|
||||
# External modules that need to be built
|
||||
ext_modules = [ Extension('nilmdb.server.rocket', ['nilmdb/server/rocket.c' ]) ]
|
||||
|
||||
# Use Cython.
|
||||
cython_modules = [ 'nilmdb.server.interval', 'nilmdb.server.rbtree' ]
|
||||
import Cython
|
||||
from Cython.Build import cythonize
|
||||
for modulename in cython_modules:
|
||||
filename = modulename.replace('.','/')
|
||||
if use_cython:
|
||||
ext_modules.extend(cythonize(filename + ".pyx"))
|
||||
else:
|
||||
cfile = filename + ".c"
|
||||
if not os.path.exists(cfile):
|
||||
raise Exception("Missing source file " + cfile + ". "
|
||||
"Try installing cython >= 0.16.")
|
||||
ext_modules.append(Extension(modulename, [ cfile ]))
|
||||
|
||||
# Get list of requirements to use in `install_requires` below. Note
|
||||
# that we don't make a distinction between things that are actually
|
||||
# required for end-users vs developers (or use `test_requires` or
|
||||
# anything else) -- just install everything for simplicity.
|
||||
install_requires = open('requirements.txt').readlines()
|
||||
# We need a MANIFEST.in. Generate it here rather than polluting the
|
||||
# repository with yet another setup-related file.
|
||||
with open("MANIFEST.in", "w") as m:
|
||||
m.write("""
|
||||
# Root
|
||||
include README.txt
|
||||
include setup.cfg
|
||||
include setup.py
|
||||
include versioneer.py
|
||||
include Makefile
|
||||
include .coveragerc
|
||||
include .pylintrc
|
||||
|
||||
# Cython files -- include source.
|
||||
recursive-include nilmdb/server *.pyx *.pyxdep *.pxd
|
||||
|
||||
# Tests
|
||||
recursive-include tests *.py
|
||||
recursive-include tests/data *
|
||||
include tests/test.order
|
||||
|
||||
# Docs
|
||||
recursive-include docs Makefile *.md
|
||||
|
||||
# Extras
|
||||
recursive-include extras *
|
||||
""")
|
||||
|
||||
# Run setup
|
||||
setup(name='nilmdb',
|
||||
version = versioneer.get_version(),
|
||||
cmdclass = versioneer.get_cmdclass(),
|
||||
url = 'https://git.jim.sh/nilm/nilmdb.git',
|
||||
url = 'https://git.jim.sh/jim/lees/nilmdb.git',
|
||||
author = 'Jim Paris',
|
||||
description = "NILM Database",
|
||||
long_description = "NILM Database",
|
||||
license = "Proprietary",
|
||||
author_email = 'jim@jtan.com',
|
||||
setup_requires = [ 'setuptools' ],
|
||||
install_requires = install_requires,
|
||||
tests_require = [ 'nose',
|
||||
'coverage',
|
||||
'numpy',
|
||||
],
|
||||
setup_requires = [ 'setuptools',
|
||||
],
|
||||
install_requires = [ 'decorator',
|
||||
'cherrypy >= 3.2',
|
||||
'simplejson',
|
||||
'python-dateutil',
|
||||
'pytz',
|
||||
'psutil >= 0.3.0',
|
||||
'requests >= 1.1.0',
|
||||
'progressbar >= 2.2',
|
||||
],
|
||||
packages = [ 'nilmdb',
|
||||
'nilmdb.utils',
|
||||
'nilmdb.utils.datetime_tz',
|
||||
'nilmdb.server',
|
||||
'nilmdb.client',
|
||||
'nilmdb.cmdline',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# comments are cool? what if they contain →UNICODEâ†<C3A2> or invalid utf-8 like Ã(
|
||||
# comments are cool?
|
||||
2.66568e+05 2.24029e+05 5.16140e+03 2.52517e+03 8.35084e+03 3.72470e+03 1.35534e+03 2.03900e+03
|
||||
2.57914e+05 2.27183e+05 4.30368e+03 4.13080e+03 7.25535e+03 4.89047e+03 1.63859e+03 1.93496e+03
|
||||
2.51717e+05 2.26047e+05 5.99445e+03 3.49363e+03 8.07250e+03 5.08267e+03 2.26917e+03 2.86231e+03
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
hi
|
||||
|
|
@ -1 +0,0 @@
|
|||
hi
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
hi
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
hi
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user