Compare commits

...

18 Commits

Author SHA1 Message Date
eae6dd623f Freeze requirements 2020-08-06 18:13:01 -04:00
09a9ed9734 Fix #! at top of shell scripts for py3 and venvs 2020-08-05 17:01:47 -04:00
079a2b5192 Fix nilmrun scripts for python 3 2020-08-05 16:59:56 -04:00
e7f52a4013 Update versioneer 2020-08-05 16:42:46 -04:00
c36b9b97e0 Add missing MANIFEST.in, update .gitignore 2020-08-03 23:48:07 -04:00
549a27e66c Add flake8 tests and clean up warnings 2020-08-03 23:46:34 -04:00
cd68389e9a Update for Python 3 and psutil >= 2. 2020-08-03 23:08:52 -04:00
6faf563bda Remove nilmrun.testfilter
This module was only used to assist testing, but wasn't actually
available in a test environment where no copies of nilmrun have been
installed.  Just code the contents directly into the filters that the
tests are running.
2020-08-03 23:05:32 -04:00
58fd9d1559 Run cherrypy always in "embedded" mode
Non-embedded mode is not used in the test suite or wsgi server;
it was an option in the standalong nilmdb-server script, but it's
really not necessary, and removing it gets rid of some untested
code.
2020-08-03 16:57:17 -04:00
2bc939d42d CherryPy bug 1200 is no longer an issue 2020-08-03 16:55:23 -04:00
fe36722684 Use built-in json module rather than external simplejson 2020-08-03 16:54:47 -04:00
24740a838e Run 2to3 over all code 2020-08-03 16:54:33 -04:00
d332fa1e0f Fix inconsistent indentation 2020-08-03 16:52:02 -04:00
cbc7a7dd55 Start Python 3 conversion 2020-08-03 16:51:01 -04:00
38c3e67cf9 Fix long lines 2013-08-08 17:47:06 -04:00
7f05a0fb62 Switch to bash for Unicode tests 2013-08-07 12:42:05 -04:00
81c2ad07d4 More robust checking in 09_info test 2013-08-07 12:32:21 -04:00
3588e843ac Replace burnP5 with dd 2013-08-07 12:31:43 -04:00
22 changed files with 2390 additions and 891 deletions

2
.gitattributes vendored
View File

@ -1 +1 @@
src/_version.py export-subst nilmrun/_version.py export-subst

2
.gitignore vendored
View File

@ -3,8 +3,8 @@ build/
*.pyc *.pyc
dist/ dist/
nilmrun.egg-info/ nilmrun.egg-info/
.eggs/
# This gets generated as needed by setup.py # This gets generated as needed by setup.py
MANIFEST.in
MANIFEST MANIFEST

8
MANIFEST.in Normal file
View File

@ -0,0 +1,8 @@
# Root
include README.md
include setup.py
include versioneer.py
include Makefile
# Version
include nilmrun/_version.py

View File

@ -1,48 +1,49 @@
# By default, run the tests. # By default, run the tests.
all: test all: test
test2:
nilmrun/trainola.py data.js
version: version:
python setup.py version python3 setup.py version
build: build:
python setup.py build_ext --inplace python3 setup.py build_ext --inplace
dist: sdist dist: sdist
sdist: sdist:
python setup.py sdist python3 setup.py sdist
install: install:
python setup.py install python3 setup.py install
develop: develop:
python setup.py develop python3 setup.py develop
docs: docs:
make -C docs make -C docs
ctrl: flake
flake:
flake8 nilmrun
lint: lint:
pylint --rcfile=.pylintrc nilmdb pylint3 --rcfile=setup.cfg nilmrun
test: test:
ifeq ($(INSIDE_EMACS), t) ifneq ($(INSIDE_EMACS),)
# Use the slightly more flexible script # Use the slightly more flexible script
python setup.py build_ext --inplace python3 setup.py build_ext --inplace
python tests/runtests.py python3 tests/runtests.py
else else
# Let setup.py check dependencies, build stuff, and run the test # Let setup.py check dependencies, build stuff, and run the test
python setup.py nosetests python3 setup.py nosetests
endif endif
clean:: clean::
find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
rm -f .coverage rm -f .coverage
find . -name '*pyc' | xargs rm -f rm -rf nilmrun.egg-info/ build/
rm -rf nilmtools.egg-info/ build/ MANIFEST.in
make -C docs clean make -C docs clean
gitclean:: gitclean::
git clean -dXf git clean -dXf
.PHONY: all version dist sdist install docs lint test clean gitclean .PHONY: all version dist sdist install docs test
.PHONY: ctrl lint flake clean gitclean

32
README.md Normal file
View File

@ -0,0 +1,32 @@
# nilmrun: Run NilmDB filters
by Jim Paris <jim@jtan.com>
## Prerequisites:
# Runtime and build environments
sudo apt-get install python3
# Create a new Python virtual environment to isolate deps.
python3 -m venv ../venv
source ../venv/bin/activate # run "deactivate" to leave
# Install all Python dependencies
pip3 install -r requirements.txt
## Install:
Install it into the virtual environment
python3 setup.py install
If you want to instead install it system-wide, you will also need to
install the requirements system-wide:
sudo pip3 install -r requirements.txt
sudo python3 setup.py install
## Usage:
nilmrun-server --help
See docs/wsgi.md for info on setting up a WSGI application in Apache.

View File

@ -1,20 +0,0 @@
nilmrun: Run NilmDB filters
by Jim Paris <jim@jtan.com>
Prerequisites:
# Runtime and build environments
sudo apt-get install python2.7 python-setuptools
# Plus nilmdb and its dependencies
nilmdb (1.9.5+)
Install:
python setup.py install
Usage:
nilmrun-server --help
See docs/wsgi.md for info on setting up a WSGI application in Apache.

View File

@ -1,5 +1,3 @@
import nilmrun.processmanager
from ._version import get_versions from ._version import get_versions
__version__ = get_versions()['version'] __version__ = get_versions()['version']
del get_versions del get_versions

View File

@ -1,197 +1,520 @@
IN_LONG_VERSION_PY = True
# This file helps to compute a version number in source trees obtained from # This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag # git-archive tarball (such as those provided by githubs download-from-tag
# feature). Distribution tarballs (build by setup.py sdist) and build # feature). Distribution tarballs (built by setup.py sdist) and build
# directories (produced by setup.py build) will contain a much shorter file # directories (produced by setup.py build) will contain a much shorter file
# that just contains the computed version number. # that just contains the computed version number.
# This file is released into the public domain. Generated by # This file is released into the public domain. Generated by
# versioneer-0.7+ (https://github.com/warner/python-versioneer) # versioneer-0.18 (https://github.com/warner/python-versioneer)
# these strings will be replaced by git during git-archive
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
"""Git implementation of _version.py."""
import errno
import os
import re
import subprocess import subprocess
import sys import sys
def run_command(args, cwd=None, verbose=False):
def get_keywords():
"""Get the keywords needed to look up the version information."""
# these strings will be replaced by git during git-archive.
# setup.py/versioneer.py will grep for the variable names, so they must
# each be defined on a line of their own. _version.py will just call
# get_keywords().
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
git_date = "$Format:%ci$"
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
return keywords
class VersioneerConfig:
"""Container for Versioneer configuration parameters."""
def get_config():
"""Create, populate and return the VersioneerConfig() object."""
# these strings are filled in when 'setup.py versioneer' creates
# _version.py
cfg = VersioneerConfig()
cfg.VCS = "git"
cfg.style = "pep440"
cfg.tag_prefix = "nilmrun-"
cfg.parentdir_prefix = "nilmrun-"
cfg.versionfile_source = "nilmrun/_version.py"
cfg.verbose = False
return cfg
class NotThisMethod(Exception):
"""Exception raised if a method is not valid for the current scenario."""
LONG_VERSION_PY = {}
HANDLERS = {}
def register_vcs_handler(vcs, method): # decorator
"""Decorator to mark a method as the handler for a particular VCS."""
def decorate(f):
"""Store f in HANDLERS[vcs][method]."""
if vcs not in HANDLERS:
HANDLERS[vcs] = {}
HANDLERS[vcs][method] = f
return f
return decorate
def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
env=None):
"""Call the given command(s)."""
assert isinstance(commands, list)
p = None
for c in commands:
try: try:
dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git # remember shell=False, so use git.cmd on windows, not just git
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) p = subprocess.Popen([c] + args, cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
break
except EnvironmentError: except EnvironmentError:
e = sys.exc_info()[1] e = sys.exc_info()[1]
if e.errno == errno.ENOENT:
continue
if verbose: if verbose:
print("unable to run %s" % args[0]) print("unable to run %s" % dispcmd)
print(e) print(e)
return None return None, None
else:
if verbose:
print("unable to find command, tried %s" % (commands,))
return None, None
stdout = p.communicate()[0].strip() stdout = p.communicate()[0].strip()
if sys.version >= '3': if sys.version_info[0] >= 3:
stdout = stdout.decode() stdout = stdout.decode()
if p.returncode != 0: if p.returncode != 0:
if verbose: if verbose:
print("unable to run %s (error)" % args[0]) print("unable to run %s (error)" % dispcmd)
return None print("stdout was %s" % stdout)
return stdout return None, p.returncode
return stdout, p.returncode
import sys def versions_from_parentdir(parentdir_prefix, root, verbose):
import re """Try to determine the version from the parent directory name.
import os.path
def get_expanded_variables(versionfile_source): Source tarballs conventionally unpack into a directory that includes both
the project name and a version string. We will also support searching up
two directory levels for an appropriately named parent directory
"""
rootdirs = []
for i in range(3):
dirname = os.path.basename(root)
if dirname.startswith(parentdir_prefix):
return {"version": dirname[len(parentdir_prefix):],
"full-revisionid": None,
"dirty": False, "error": None, "date": None}
else:
rootdirs.append(root)
root = os.path.dirname(root) # up a level
if verbose:
print("Tried directories %s but none started with prefix %s" %
(str(rootdirs), parentdir_prefix))
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
@register_vcs_handler("git", "get_keywords")
def git_get_keywords(versionfile_abs):
"""Extract version information from the given file."""
# the code embedded in _version.py can just fetch the value of these # the code embedded in _version.py can just fetch the value of these
# variables. When used from setup.py, we don't want to import # keywords. When used from setup.py, we don't want to import _version.py,
# _version.py, so we do it with a regexp instead. This function is not # so we do it with a regexp instead. This function is not used from
# used from _version.py. # _version.py.
variables = {} keywords = {}
try: try:
for line in open(versionfile_source,"r").readlines(): f = open(versionfile_abs, "r")
for line in f.readlines():
if line.strip().startswith("git_refnames ="): if line.strip().startswith("git_refnames ="):
mo = re.search(r'=\s*"(.*)"', line) mo = re.search(r'=\s*"(.*)"', line)
if mo: if mo:
variables["refnames"] = mo.group(1) keywords["refnames"] = mo.group(1)
if line.strip().startswith("git_full ="): if line.strip().startswith("git_full ="):
mo = re.search(r'=\s*"(.*)"', line) mo = re.search(r'=\s*"(.*)"', line)
if mo: if mo:
variables["full"] = mo.group(1) keywords["full"] = mo.group(1)
if line.strip().startswith("git_date ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["date"] = mo.group(1)
f.close()
except EnvironmentError: except EnvironmentError:
pass pass
return variables return keywords
def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
refnames = variables["refnames"].strip() @register_vcs_handler("git", "keywords")
def git_versions_from_keywords(keywords, tag_prefix, verbose):
"""Get version information from git keywords."""
if not keywords:
raise NotThisMethod("no keywords at all, weird")
date = keywords.get("date")
if date is not None:
# git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
# datestamp. However we prefer "%ci" (which expands to an "ISO-8601
# -like" string, which we must then edit to make compliant), because
# it's been around since git-1.5.3, and it's too difficult to
# discover which version we're using, or to work around using an
# older one.
date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
refnames = keywords["refnames"].strip()
if refnames.startswith("$Format"): if refnames.startswith("$Format"):
if verbose: if verbose:
print("variables are unexpanded, not using") print("keywords are unexpanded, not using")
return {} # unexpanded, so not in an unpacked git-archive tarball raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
refs = set([r.strip() for r in refnames.strip("()").split(",")]) refs = set([r.strip() for r in refnames.strip("()").split(",")])
for ref in list(refs): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
if not re.search(r'\d', ref): # just "foo-1.0". If we see a "tag: " prefix, prefer those.
TAG = "tag: "
tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
if not tags:
# Either we're using git < 1.8.3, or there really are no tags. We use
# a heuristic: assume all version tags have a digit. The old git %d
# expansion behaves like git log --decorate=short and strips out the
# refs/heads/ and refs/tags/ prefixes that would let us distinguish
# between branches and tags. By ignoring refnames without digits, we
# filter out many common branch names like "release" and
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose: if verbose:
print("discarding '%s', no digits" % ref) print("discarding '%s', no digits" % ",".join(refs - tags))
refs.discard(ref)
# Assume all version tags have a digit. git's %d expansion
# behaves like git log --decorate=short and strips out the
# refs/heads/ and refs/tags/ prefixes that would let us
# distinguish between branches and tags. By ignoring refnames
# without digits, we filter out many common branch names like
# "release" and "stabilization", as well as "HEAD" and "master".
if verbose: if verbose:
print("remaining refs: %s" % ",".join(sorted(refs))) print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(refs): for ref in sorted(tags):
# sorting will prefer e.g. "2.0" over "2.0rc1" # sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix): if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):] r = ref[len(tag_prefix):]
if verbose: if verbose:
print("picking %s" % r) print("picking %s" % r)
return { "version": r, return {"version": r,
"full": variables["full"].strip() } "full-revisionid": keywords["full"].strip(),
# no suitable tags, so we use the full revision id "dirty": False, "error": None,
"date": date}
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose: if verbose:
print("no suitable tags, using full revision id") print("no suitable tags, using unknown + full revision id")
return { "version": variables["full"].strip(), return {"version": "0+unknown",
"full": variables["full"].strip() } "full-revisionid": keywords["full"].strip(),
"dirty": False, "error": "no suitable tags", "date": None}
def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
# this runs 'git' from the root of the source tree. That either means
# someone ran a setup.py command (and this code is in versioneer.py, so
# IN_LONG_VERSION_PY=False, thus the containing directory is the root of
# the source tree), or someone ran a project-specific entry point (and
# this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
# containing directory is somewhere deeper in the source tree). This only
# gets called if the git-archive 'subst' variables were *not* expanded,
# and _version.py hasn't already been rewritten with a short version
# string, meaning we're inside a checked out source tree.
try: @register_vcs_handler("git", "pieces_from_vcs")
here = os.path.abspath(__file__) def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
except NameError: """Get version from 'git describe' in the root of the source tree.
# some py2exe/bbfreeze/non-CPython implementations don't do __file__
return {} # not always correct
# versionfile_source is the relative path from the top of the source tree This only gets called if the git-archive 'subst' keywords were *not*
# (where the .git directory might live) to this file. Invert this to find expanded, and _version.py hasn't already been rewritten with a short
# the root from __file__. version string, meaning we're inside a checked out source tree.
root = here """
if IN_LONG_VERSION_PY: GITS = ["git"]
for i in range(len(versionfile_source.split("/"))):
root = os.path.dirname(root)
else:
root = os.path.dirname(here)
if not os.path.exists(os.path.join(root, ".git")):
if verbose:
print("no .git in %s" % root)
return {}
GIT = "git"
if sys.platform == "win32": if sys.platform == "win32":
GIT = "git.cmd" GITS = ["git.cmd", "git.exe"]
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
hide_stderr=True)
if rc != 0:
if verbose:
print("Directory %s not under git control" % root)
raise NotThisMethod("'git rev-parse --git-dir' returned error")
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
# if there isn't one, this yields HEX[-dirty] (no NUM)
describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
"--always", "--long",
"--match", "%s*" % tag_prefix],
cwd=root) cwd=root)
if stdout is None: # --long was added in git-1.5.5
return {} if describe_out is None:
if not stdout.startswith(tag_prefix): raise NotThisMethod("'git describe' failed")
describe_out = describe_out.strip()
full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
if full_out is None:
raise NotThisMethod("'git rev-parse' failed")
full_out = full_out.strip()
pieces = {}
pieces["long"] = full_out
pieces["short"] = full_out[:7] # maybe improved later
pieces["error"] = None
# parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
# TAG might have hyphens.
git_describe = describe_out
# look for -dirty suffix
dirty = git_describe.endswith("-dirty")
pieces["dirty"] = dirty
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
# now we have TAG-NUM-gHEX or HEX
if "-" in git_describe:
# TAG-NUM-gHEX
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
if not mo:
# unparseable. Maybe git-describe is misbehaving?
pieces["error"] = ("unable to parse git-describe output: '%s'"
% describe_out)
return pieces
# tag
full_tag = mo.group(1)
if not full_tag.startswith(tag_prefix):
if verbose: if verbose:
print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) fmt = "tag '%s' doesn't start with prefix '%s'"
return {} print(fmt % (full_tag, tag_prefix))
tag = stdout[len(tag_prefix):] pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) % (full_tag, tag_prefix))
if stdout is None: return pieces
return {} pieces["closest-tag"] = full_tag[len(tag_prefix):]
full = stdout.strip()
if tag.endswith("-dirty"):
full += "-dirty"
return {"version": tag, "full": full}
# distance: number of commits since tag
pieces["distance"] = int(mo.group(2))
# commit: short hex revision ID
pieces["short"] = mo.group(3)
def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
if IN_LONG_VERSION_PY:
# We're running from _version.py. If it's from a source tree
# (execute-in-place), we can work upwards to find the root of the
# tree, and then check the parent directory for a version string. If
# it's in an installed application, there's no hope.
try:
here = os.path.abspath(__file__)
except NameError:
# py2exe/bbfreeze/non-CPython don't have __file__
return {} # without __file__, we have no hope
# versionfile_source is the relative path from the top of the source
# tree to _version.py. Invert this to find the root from __file__.
root = here
for i in range(len(versionfile_source.split("/"))):
root = os.path.dirname(root)
else: else:
# we're running from versioneer.py, which means we're running from # HEX: no tags
# the setup.py in a source tree. sys.argv[0] is setup.py in the root. pieces["closest-tag"] = None
here = os.path.abspath(sys.argv[0]) count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
root = os.path.dirname(here) cwd=root)
pieces["distance"] = int(count_out) # total number of commits
# Source tarballs conventionally unpack into a directory that includes # commit date: see ISO-8601 comment in git_versions_from_keywords()
# both the project name and a version string. date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
dirname = os.path.basename(root) cwd=root)[0].strip()
if not dirname.startswith(parentdir_prefix): pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
if verbose:
print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
(root, dirname, parentdir_prefix))
return None
return {"version": dirname[len(parentdir_prefix):], "full": ""}
tag_prefix = "nilmrun-" return pieces
parentdir_prefix = "nilmrun-"
versionfile_source = "nilmrun/_version.py"
def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
variables = { "refnames": git_refnames, "full": git_full } def plus_or_dot(pieces):
ver = versions_from_expanded_variables(variables, tag_prefix, verbose) """Return a + if we don't already have one, else return a ."""
if not ver: if "+" in pieces.get("closest-tag", ""):
ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) return "."
if not ver: return "+"
ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
def render_pep440(pieces):
"""Build up version string, with post-release "local version identifier".
Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
Exceptions:
1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += plus_or_dot(pieces)
rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
else:
# exception #1
rendered = "0+untagged.%d.g%s" % (pieces["distance"],
pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
return rendered
def render_pep440_pre(pieces):
"""TAG[.post.devDISTANCE] -- No -dirty.
Exceptions:
1: no tags. 0.post.devDISTANCE
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"]:
rendered += ".post.dev%d" % pieces["distance"]
else:
# exception #1
rendered = "0.post.dev%d" % pieces["distance"]
return rendered
def render_pep440_post(pieces):
"""TAG[.postDISTANCE[.dev0]+gHEX] .
The ".dev0" means dirty. Note that .dev0 sorts backwards
(a dirty tree will appear "older" than the corresponding clean one),
but you shouldn't be releasing software with -dirty anyways.
Exceptions:
1: no tags. 0.postDISTANCE[.dev0]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += ".post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
rendered += plus_or_dot(pieces)
rendered += "g%s" % pieces["short"]
else:
# exception #1
rendered = "0.post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
rendered += "+g%s" % pieces["short"]
return rendered
def render_pep440_old(pieces):
"""TAG[.postDISTANCE[.dev0]] .
The ".dev0" means dirty.
Eexceptions:
1: no tags. 0.postDISTANCE[.dev0]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += ".post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
else:
# exception #1
rendered = "0.post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
return rendered
def render_git_describe(pieces):
"""TAG[-DISTANCE-gHEX][-dirty].
Like 'git describe --tags --dirty --always'.
Exceptions:
1: no tags. HEX[-dirty] (note: no 'g' prefix)
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"]:
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
else:
# exception #1
rendered = pieces["short"]
if pieces["dirty"]:
rendered += "-dirty"
return rendered
def render_git_describe_long(pieces):
"""TAG-DISTANCE-gHEX[-dirty].
Like 'git describe --tags --dirty --always -long'.
The distance/hash is unconditional.
Exceptions:
1: no tags. HEX[-dirty] (note: no 'g' prefix)
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
else:
# exception #1
rendered = pieces["short"]
if pieces["dirty"]:
rendered += "-dirty"
return rendered
def render(pieces, style):
"""Render the given version pieces into the requested style."""
if pieces["error"]:
return {"version": "unknown",
"full-revisionid": pieces.get("long"),
"dirty": None,
"error": pieces["error"],
"date": None}
if not style or style == "default":
style = "pep440" # the default
if style == "pep440":
rendered = render_pep440(pieces)
elif style == "pep440-pre":
rendered = render_pep440_pre(pieces)
elif style == "pep440-post":
rendered = render_pep440_post(pieces)
elif style == "pep440-old":
rendered = render_pep440_old(pieces)
elif style == "git-describe":
rendered = render_git_describe(pieces)
elif style == "git-describe-long":
rendered = render_git_describe_long(pieces)
else:
raise ValueError("unknown style '%s'" % style)
return {"version": rendered, "full-revisionid": pieces["long"],
"dirty": pieces["dirty"], "error": None,
"date": pieces.get("date")}
def get_versions():
"""Get version information or return default if unable to do so."""
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
# __file__, we can work backwards from there to the root. Some
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
# case we can only use expanded keywords.
cfg = get_config()
verbose = cfg.verbose
try:
return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
verbose) verbose)
if not ver: except NotThisMethod:
ver = default pass
return ver
try:
root = os.path.realpath(__file__)
# versionfile_source is the relative path from the top of the source
# tree (where the .git directory might live) to this file. Invert
# this to find the root from __file__.
for i in cfg.versionfile_source.split('/'):
root = os.path.dirname(root)
except NameError:
return {"version": "0+unknown", "full-revisionid": None,
"dirty": None,
"error": "unable to find root of source tree",
"date": None}
try:
pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
return render(pieces, cfg.style)
except NotThisMethod:
pass
try:
if cfg.parentdir_prefix:
return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
except NotThisMethod:
pass
return {"version": "0+unknown", "full-revisionid": None,
"dirty": None,
"error": "unable to compute version", "date": None}

View File

@ -1,10 +1,8 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.utils.printf import *
import threading import threading
import subprocess import subprocess
import cStringIO import io
import sys import sys
import os import os
import signal import signal
@ -15,16 +13,18 @@ import tempfile
import atexit import atexit
import shutil import shutil
class ProcessError(Exception): class ProcessError(Exception):
pass pass
class LogReceiver(object): class LogReceiver(object):
"""Spawn a thread that listens to a pipe for log messages, """Spawn a thread that listens to a pipe for log messages,
and stores them locally.""" and stores them locally."""
def __init__(self, pipe): def __init__(self, pipe):
self.pipe = pipe self.pipe = pipe
self.log = cStringIO.StringIO() self.log = io.BytesIO()
self.thread = threading.Thread(target = self.run) self.thread = threading.Thread(target=self.run)
self.thread.start() self.thread.start()
def run(self): def run(self):
@ -39,11 +39,12 @@ class LogReceiver(object):
return self.log.getvalue() return self.log.getvalue()
def clear(self): def clear(self):
self.log = cStringIO.StringIO() self.log = io.BytesIO()
class Process(object): class Process(object):
"""Spawn and manage a subprocess, and capture its output.""" """Spawn and manage a subprocess, and capture its output."""
def __init__(self, argv, tempfile = None): def __init__(self, argv, tempfile=None):
self.start_time = None self.start_time = None
# Use a pipe for communicating log data # Use a pipe for communicating log data
@ -55,9 +56,9 @@ class Process(object):
# Spawn the new process # Spawn the new process
try: try:
self._process = subprocess.Popen(args = argv, stdin = nullfd, self._process = subprocess.Popen(args=argv, stdin=nullfd,
stdout = wpipe, stderr = wpipe, stdout=wpipe, stderr=wpipe,
close_fds = True, cwd = "/tmp") close_fds=True, cwd="/tmp")
except (OSError, TypeError) as e: except (OSError, TypeError) as e:
raise ProcessError(str(e)) raise ProcessError(str(e))
finally: finally:
@ -69,7 +70,7 @@ class Process(object):
self.start_time = time.time() self.start_time = time.time()
self.pid = str(uuid.uuid1(self._process.pid or 0)) self.pid = str(uuid.uuid1(self._process.pid or 0))
def _join(self, timeout = 1.0): def _join(self, timeout=1.0):
start = time.time() start = time.time()
while True: while True:
if self._process.poll() is not None: if self._process.poll() is not None:
@ -78,7 +79,7 @@ class Process(object):
return False return False
time.sleep(0.1) time.sleep(0.1)
def terminate(self, timeout = 1.0): def terminate(self, timeout=1.0):
"""Terminate a process, and all of its children that are in the same """Terminate a process, and all of its children that are in the same
process group.""" process group."""
try: try:
@ -101,7 +102,7 @@ class Process(object):
# Find all children # Find all children
group = getpgid(self._process.pid) group = getpgid(self._process.pid)
main = psutil.Process(self._process.pid) main = psutil.Process(self._process.pid)
allproc = [ main ] + main.get_children(recursive = True) allproc = [main] + main.children(recursive=True)
# Kill with SIGTERM, if they're still in this process group # Kill with SIGTERM, if they're still in this process group
for proc in allproc: for proc in allproc:
@ -142,22 +143,22 @@ class Process(object):
Call .get_info() about a second later.""" Call .get_info() about a second later."""
try: try:
main = psutil.Process(self._process.pid) main = psutil.Process(self._process.pid)
self._process_list = [ main ] + main.get_children(recursive = True) self._process_list = [main] + main.children(recursive=True)
for proc in self._process_list: for proc in self._process_list:
proc.get_cpu_percent(0) proc.cpu_percent(0)
except psutil.Error: # pragma: no cover (race condition) except psutil.Error: # pragma: no cover (race condition)
self._process_list = [ ] self._process_list = []
@staticmethod @staticmethod
def get_empty_info(): def get_empty_info():
return { "cpu_percent": 0, return {"cpu_percent": 0,
"cpu_user": 0, "cpu_user": 0,
"cpu_sys": 0, "cpu_sys": 0,
"mem_phys": 0, "mem_phys": 0,
"mem_virt": 0, "mem_virt": 0,
"io_read": 0, "io_read": 0,
"io_write": 0, "io_write": 0,
"procs": 0 } "procs": 0}
def get_info(self): def get_info(self):
"""Return a dictionary with info about the process CPU and memory """Return a dictionary with info about the process CPU and memory
@ -165,14 +166,14 @@ class Process(object):
d = self.get_empty_info() d = self.get_empty_info()
for proc in self._process_list: for proc in self._process_list:
try: try:
d["cpu_percent"] += proc.get_cpu_percent(0) d["cpu_percent"] += proc.cpu_percent(0)
cpuinfo = proc.get_cpu_times() cpuinfo = proc.cpu_times()
d["cpu_user"] += cpuinfo.user d["cpu_user"] += cpuinfo.user
d["cpu_sys"] += cpuinfo.system d["cpu_sys"] += cpuinfo.system
meminfo = proc.get_memory_info() meminfo = proc.memory_info()
d["mem_phys"] += meminfo.rss d["mem_phys"] += meminfo.rss
d["mem_virt"] += meminfo.vms d["mem_virt"] += meminfo.vms
ioinfo = proc.get_io_counters() ioinfo = proc.io_counters()
d["io_read"] += ioinfo.read_bytes d["io_read"] += ioinfo.read_bytes
d["io_write"] += ioinfo.write_bytes d["io_write"] += ioinfo.write_bytes
d["procs"] += 1 d["procs"] += 1
@ -180,6 +181,7 @@ class Process(object):
pass pass
return d return d
class ProcessManager(object): class ProcessManager(object):
"""Track and manage a collection of Process objects""" """Track and manage a collection of Process objects"""
def __init__(self): def __init__(self):
@ -197,7 +199,7 @@ class ProcessManager(object):
def _atexit(self): def _atexit(self):
# Kill remaining processes, remove their dirs # Kill remaining processes, remove their dirs
for pid in self.processes.keys(): for pid in list(self.processes.keys()):
try: try:
self.processes[pid].terminate() self.processes[pid].terminate()
del self.processes[pid] del self.processes[pid]
@ -207,7 +209,7 @@ class ProcessManager(object):
pass pass
def __iter__(self): def __iter__(self):
return iter(self.processes.keys()) return iter(list(self.processes.keys()))
def __getitem__(self, key): def __getitem__(self, key):
return self.processes[key] return self.processes[key]
@ -218,7 +220,7 @@ class ProcessManager(object):
accessible in the code as sys.argv[1:].""" accessible in the code as sys.argv[1:]."""
# The easiest way to do this, by far, is to just write the # The easiest way to do this, by far, is to just write the
# code to a file. Make a directory to put it in. # code to a file. Make a directory to put it in.
tmpdir = tempfile.mkdtemp(prefix = "nilmrun-usercode-") tmpdir = tempfile.mkdtemp(prefix="nilmrun-usercode-")
try: try:
# Write the code # Write the code
codepath = os.path.join(tmpdir, "usercode.py") codepath = os.path.join(tmpdir, "usercode.py")
@ -229,7 +231,7 @@ class ProcessManager(object):
f.write(repr(args)) f.write(repr(args))
# Run the code # Run the code
argv = [ sys.executable, "-B", "-s", "-u", codepath ] + args argv = [sys.executable, "-B", "-s", "-u", codepath] + args
pid = self.run_command(argv) pid = self.run_command(argv)
# Save the temp dir # Save the temp dir
@ -260,28 +262,27 @@ class ProcessManager(object):
def get_info(self): def get_info(self):
"""Get info about all running PIDs""" """Get info about all running PIDs"""
info = { "total" : Process.get_empty_info(), info = {
"pids" : {}, "total": Process.get_empty_info(),
"system" : {} "pids": {},
"system": {}
} }
# Trigger CPU usage collection # Trigger CPU usage collection
for pid in self: for pid in self:
self[pid].get_info_prepare() self[pid].get_info_prepare()
psutil.cpu_percent(0, percpu = True) psutil.cpu_percent(0, percpu=True)
# Give it some time # Give it some time
time.sleep(1) time.sleep(1)
# Retrieve info for system # Retrieve info for system
info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True)) info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True))
info["system"]["cpu_max"] = 100.0 * psutil.NUM_CPUS info["system"]["cpu_max"] = 100.0 * psutil.cpu_count()
info["system"]["procs"] = len(psutil.get_pid_list()) info["system"]["procs"] = len(psutil.pids())
# psutil > 0.6.0's psutil.virtual_memory() would be better here, meminfo = psutil.virtual_memory()
# but this should give the same info.
meminfo = psutil.phymem_usage()
info["system"]["mem_total"] = meminfo.total info["system"]["mem_total"] = meminfo.total
info["system"]["mem_used"] = int(meminfo.total * meminfo.percent / 100) info["system"]["mem_used"] = meminfo.used
# Retrieve info for each PID # Retrieve info for each PID
for pid in self: for pid in self:

View File

@ -1,19 +1,12 @@
"""CherryPy-based server for running NILM filters via HTTP""" """CherryPy-based server for running NILM filters via HTTP"""
import cherrypy import cherrypy
import sys
import os import os
import socket import socket
import simplejson as json
import traceback import traceback
import time
import nilmdb from nilmdb.utils.printf import sprintf
from nilmdb.utils.printf import *
from nilmdb.server.serverutil import ( from nilmdb.server.serverutil import (
chunked_response,
response_type,
workaround_cp_bug_1200,
exception_to_httperror, exception_to_httperror,
CORS_allow, CORS_allow,
json_to_request_params, json_to_request_params,
@ -24,11 +17,12 @@ from nilmdb.server.serverutil import (
) )
from nilmdb.utils import serializer_proxy from nilmdb.utils import serializer_proxy
import nilmrun import nilmrun
import nilmrun.testfilter import nilmrun.processmanager
# Add CORS_allow tool # Add CORS_allow tool
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow) cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
# CherryPy apps # CherryPy apps
class App(object): class App(object):
"""Root application for NILM runner""" """Root application for NILM runner"""
@ -55,6 +49,7 @@ class App(object):
def version(self): def version(self):
return nilmrun.__version__ return nilmrun.__version__
class AppProcess(object): class AppProcess(object):
def __init__(self, manager): def __init__(self, manager):
@ -76,7 +71,7 @@ class AppProcess(object):
# /process/status # /process/status
@cherrypy.expose @cherrypy.expose
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
def status(self, pid, clear = False): def status(self, pid, clear=False):
"""Return status about a process. If clear = True, also clear """Return status about a process. If clear = True, also clear
the log.""" the log."""
clear = bool_param(clear) clear = bool_param(clear)
@ -106,7 +101,7 @@ class AppProcess(object):
@cherrypy.expose @cherrypy.expose
@cherrypy.tools.json_in() @cherrypy.tools.json_in()
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
@cherrypy.tools.CORS_allow(methods = ["POST"]) @cherrypy.tools.CORS_allow(methods=["POST"])
def remove(self, pid): def remove(self, pid):
"""Remove a process from the manager, killing it if necessary.""" """Remove a process from the manager, killing it if necessary."""
if pid not in self.manager: if pid not in self.manager:
@ -118,6 +113,7 @@ class AppProcess(object):
self.manager.remove(pid) self.manager.remove(pid)
return status return status
class AppRun(object): class AppRun(object):
def __init__(self, manager): def __init__(self, manager):
self.manager = manager self.manager = manager
@ -127,7 +123,7 @@ class AppRun(object):
@cherrypy.tools.json_in() @cherrypy.tools.json_in()
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError) @exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods = ["POST"]) @cherrypy.tools.CORS_allow(methods=["POST"])
def command(self, argv): def command(self, argv):
"""Execute an arbitrary program on the server. argv is a """Execute an arbitrary program on the server. argv is a
list of the program and its arguments: 'argv[0]' is the program list of the program and its arguments: 'argv[0]' is the program
@ -142,8 +138,8 @@ class AppRun(object):
@cherrypy.tools.json_in() @cherrypy.tools.json_in()
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError) @exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods = ["POST"]) @cherrypy.tools.CORS_allow(methods=["POST"])
def code(self, code, args = None): def code(self, code, args=None):
"""Execute arbitrary Python code. 'code' is a formatted string. """Execute arbitrary Python code. 'code' is a formatted string.
It will be run as if it were written into a Python file and It will be run as if it were written into a Python file and
executed. 'args' is a list of strings, and they are passed executed. 'args' is a list of strings, and they are passed
@ -156,23 +152,21 @@ class AppRun(object):
"args must be a list of strings") "args must be a list of strings")
return self.manager.run_code(code, args) return self.manager.run_code(code, args)
class Server(object): class Server(object):
def __init__(self, host = '127.0.0.1', port = 8080, def __init__(self, host='127.0.0.1', port=8080,
embedded = True, # hide diagnostics and output, etc force_traceback=False, # include traceback in all errors
force_traceback = False, # include traceback in all errors basepath='', # base URL path for cherrypy.tree
basepath = '', # base URL path for cherrypy.tree
): ):
self.embedded = embedded
# Build up global server configuration # Build up global server configuration
cherrypy.config.update({ cherrypy.config.update({
'environment': 'embedded',
'server.socket_host': host, 'server.socket_host': host,
'server.socket_port': port, 'server.socket_port': port,
'engine.autoreload_on': False, 'engine.autoreload_on': False,
'server.max_request_body_size': 8*1024*1024, 'server.max_request_body_size': 8*1024*1024,
}) })
if self.embedded:
cherrypy.config.update({ 'environment': 'embedded' })
# Build up application specific configuration # Build up application specific configuration
app_config = {} app_config = {}
@ -181,23 +175,25 @@ class Server(object):
}) })
# Some default headers to just help identify that things are working # Some default headers to just help identify that things are working
app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' }) app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
# Set up Cross-Origin Resource Sharing (CORS) handler so we # Set up Cross-Origin Resource Sharing (CORS) handler so we
# can correctly respond to browsers' CORS preflight requests. # can correctly respond to browsers' CORS preflight requests.
# This also limits verbs to GET and HEAD by default. # This also limits verbs to GET and HEAD by default.
app_config.update({ 'tools.CORS_allow.on': True, app_config.update({
'tools.CORS_allow.methods': ['GET', 'HEAD'] }) 'tools.CORS_allow.on': True,
'tools.CORS_allow.methods': ['GET', 'HEAD']
})
# Configure the 'json_in' tool to also allow other content-types # Configure the 'json_in' tool to also allow other content-types
# (like x-www-form-urlencoded), and to treat JSON as a dict that # (like x-www-form-urlencoded), and to treat JSON as a dict that
# fills requests.param. # fills requests.param.
app_config.update({ 'tools.json_in.force': False, app_config.update({'tools.json_in.force': False,
'tools.json_in.processor': json_to_request_params }) 'tools.json_in.processor': json_to_request_params})
# Send tracebacks in error responses. They're hidden by the # Send tracebacks in error responses. They're hidden by the
# error_page function for client errors (code 400-499). # error_page function for client errors (code 400-499).
app_config.update({ 'request.show_tracebacks' : True }) app_config.update({'request.show_tracebacks': True})
self.force_traceback = force_traceback self.force_traceback = force_traceback
# Patch CherryPy error handler to never pad out error messages. # Patch CherryPy error handler to never pad out error messages.
@ -215,7 +211,7 @@ class Server(object):
root.process = AppProcess(manager) root.process = AppProcess(manager)
root.run = AppRun(manager) root.run = AppRun(manager)
cherrypy.tree.apps = {} cherrypy.tree.apps = {}
cherrypy.tree.mount(root, basepath, config = { "/" : app_config }) cherrypy.tree.mount(root, basepath, config={"/": app_config})
# Set up the WSGI application pointer for external programs # Set up the WSGI application pointer for external programs
self.wsgi_application = cherrypy.tree self.wsgi_application = cherrypy.tree
@ -225,16 +221,19 @@ class Server(object):
return json_error_page(status, message, traceback, version, return json_error_page(status, message, traceback, version,
self.force_traceback) self.force_traceback)
def start(self, blocking = False, event = None): def start(self, blocking=False, event=None):
cherrypy_start(blocking, event, self.embedded) cherrypy_start(blocking, event)
def stop(self): def stop(self):
cherrypy_stop() cherrypy_stop()
# Multiple processes and threads should be OK here, but we'll still # Multiple processes and threads should be OK here, but we'll still
# follow the NilmDB approach of having just one globally initialized # follow the NilmDB approach of having just one globally initialized
# copy of the server object. # copy of the server object.
_wsgi_server = None _wsgi_server = None
def wsgi_application(basepath): # pragma: no cover def wsgi_application(basepath): # pragma: no cover
"""Return a WSGI application object. """Return a WSGI application object.
@ -248,13 +247,11 @@ def wsgi_application(basepath): # pragma: no cover
# Try to start the server # Try to start the server
try: try:
_wsgi_server = nilmrun.server.Server( _wsgi_server = nilmrun.server.Server(
embedded = True, basepath=basepath.rstrip('/'))
basepath = basepath.rstrip('/'))
except Exception: except Exception:
# Build an error message on failure # Build an error message on failure
import pprint import pprint
err = sprintf("Initializing nilmrun failed:\n\n", err = "Initializing nilmrun failed:\n\n"
dbpath)
err += traceback.format_exc() err += traceback.format_exc()
try: try:
import pwd import pwd
@ -269,8 +266,10 @@ def wsgi_application(basepath): # pragma: no cover
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ)) err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
if _wsgi_server is None: if _wsgi_server is None:
# Serve up the error with our own mini WSGI app. # Serve up the error with our own mini WSGI app.
headers = [ ('Content-type', 'text/plain'), headers = [
('Content-length', str(len(err))) ] ('Content-type', 'text/plain'),
('Content-length', str(len(err)))
]
start_response("500 Internal Server Error", headers) start_response("500 Internal Server Error", headers)
return [err] return [err]

View File

@ -1,22 +0,0 @@
#!/usr/bin/python
from nilmdb.utils.printf import *
import time
import signal
import sys
# This is just for testing the process management.
def test(n):
n = int(n)
if n < 0: # raise an exception
raise Exception("test exception")
if n == 0: # ignore SIGTERM and count to 100
n = 100
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for x in range(n):
s = sprintf("dummy %d\n", x)
if x & 1:
sys.stdout.write(s)
else:
sys.stderr.write(s)
time.sleep(0.1)

2
requirements.txt Normal file
View File

@ -0,0 +1,2 @@
nilmdb>=2.0.3
psutil==5.7.2

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import * from nilmdb.utils.printf import *
@ -13,8 +13,9 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/") def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Kill/remove a process from the NilmRun server', description = 'Kill/remove a process from the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
version = nilmrun.__version__) parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url', group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url) help = 'NilmRun server URL', default = def_url)

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
import nilmrun.server import nilmrun.server
import argparse import argparse
@ -10,8 +10,9 @@ def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Run the NilmRun server', description = 'Run the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
version = nilmrun.__version__) parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-a', '--address', group.add_argument('-a', '--address',
@ -34,25 +35,24 @@ def main():
embedded = False embedded = False
server = nilmrun.server.Server(host = args.address, server = nilmrun.server.Server(host = args.address,
port = args.port, port = args.port,
embedded = embedded,
force_traceback = args.traceback) force_traceback = args.traceback)
# Print info # Print info
if not args.quiet: if not args.quiet:
print "NilmRun version: %s" % nilmrun.__version__ print("NilmRun version: %s" % nilmrun.__version__)
print ("Note: This server does not do any authentication! " + print(("Note: This server does not do any authentication! " +
"Anyone who can connect can run arbitrary commands.") "Anyone who can connect can run arbitrary commands."))
if args.address == '0.0.0.0' or args.address == '::': if args.address == '0.0.0.0' or args.address == '::':
host = socket.getfqdn() host = socket.getfqdn()
else: else:
host = args.address host = args.address
print "Server URL: http://%s:%d/" % ( host, args.port) print("Server URL: http://%s:%d/" % ( host, args.port))
print "----" print("----")
server.start(blocking = True) server.start(blocking = True)
if not args.quiet: if not args.quiet:
print "Shutting down" print("Shutting down")
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@ -1,8 +1,8 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import * from nilmdb.utils.printf import *
from nilmdb.utils import datetime_tz import datetime_tz
import nilmrun import nilmrun
import argparse import argparse
@ -13,8 +13,9 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/") def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'List NilmRun processes', description = 'List NilmRun processes',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
version = nilmrun.__version__) parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url', group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url) help = 'NilmRun server URL', default = def_url)

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import * from nilmdb.utils.printf import *
@ -14,8 +14,9 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/") def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Run a command on the NilmRun server', description = 'Run a command on the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
version = nilmrun.__version__) parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url', group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url) help = 'NilmRun server URL', default = def_url)
@ -30,14 +31,15 @@ def main():
help="Arguments for command") help="Arguments for command")
args = parser.parse_args() args = parser.parse_args()
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify) client = HTTPClient(baseurl=args.url, verify_ssl=not args.noverify,
post_json=True)
# Run command # Run command
pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg }) pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })
# If we're detaching, just print the PID # If we're detaching, just print the PID
if args.detach: if args.detach:
print pid print(pid)
raise SystemExit(0) raise SystemExit(0)
# Otherwise, watch the log output, and kill the process when it's done # Otherwise, watch the log output, and kill the process when it's done

View File

@ -20,3 +20,19 @@ cover-erase=1
stop=1 stop=1
verbosity=2 verbosity=2
tests=tests tests=tests
[versioneer]
VCS=git
style=pep440
versionfile_source=nilmrun/_version.py
versionfile_build=nilmrun/_version.py
tag_prefix=nilmrun-
parentdir_prefix=nilmrun-
[flake8]
exclude=_version.py
extend-ignore=E731
[pylint]
ignore=_version.py
disable=C0103,C0111,R0913,R0914

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
# To release a new version, tag it: # To release a new version, tag it:
# git tag -a nilmrun-1.1 -m "Version 1.1" # git tag -a nilmrun-1.1 -m "Version 1.1"
@ -6,66 +6,31 @@
# Then just package it up: # Then just package it up:
# python setup.py sdist # python setup.py sdist
# This is supposed to be using Distribute:
#
# distutils provides a "setup" method.
# setuptools is a set of monkeypatches on top of that.
# distribute is a particular version/implementation of setuptools.
#
# So we don't really know if this is using the old setuptools or the
# Distribute-provided version of setuptools.
import traceback
import sys import sys
import os import os
from setuptools import setup
try:
from setuptools import setup, find_packages
import distutils.version
except ImportError:
traceback.print_exc()
print "Please install the prerequisites listed in README.txt"
sys.exit(1)
# Versioneer manages version numbers from git tags. # Versioneer manages version numbers from git tags.
# https://github.com/warner/python-versioneer # https://github.com/warner/python-versioneer
import versioneer import versioneer
versioneer.versionfile_source = 'nilmrun/_version.py'
versioneer.versionfile_build = 'nilmrun/_version.py'
versioneer.tag_prefix = 'nilmrun-'
versioneer.parentdir_prefix = 'nilmrun-'
# Hack to workaround logging/multiprocessing issue: # Get list of requirements to use in `install_requires` below. Note
# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ # that we don't make a distinction between things that are actually
try: import multiprocessing # required for end-users vs developers (or use `test_requires` or
except: pass # anything else) -- just install everything for simplicity.
install_requires = open('requirements.txt').readlines()
# We need a MANIFEST.in. Generate it here rather than polluting the
# repository with yet another setup-related file.
with open("MANIFEST.in", "w") as m:
m.write("""
# Root
include README.txt
include setup.py
include versioneer.py
include Makefile
""")
# Run setup # Run setup
setup(name='nilmrun', setup(name='nilmrun',
version = versioneer.get_version(), version = versioneer.get_version(),
cmdclass = versioneer.get_cmdclass(), cmdclass = versioneer.get_cmdclass(),
url = 'https://git.jim.sh/jim/lees/nilmrun.git', url = 'https://git.jim.sh/nilm/nilmrun.git',
author = 'Jim Paris', author = 'Jim Paris',
description = "NILM Database Filter Runner", description = "NILM Database Filter Runner",
long_description = "NILM Database Filter Runner", long_description = "NILM Database Filter Runner",
license = "Proprietary", license = "Proprietary",
author_email = 'jim@jtan.com', author_email = 'jim@jtan.com',
install_requires = [ 'nilmdb >= 1.9.5', install_requires = install_requires,
'psutil >= 0.3.0',
'cherrypy >= 3.2',
'simplejson',
],
packages = [ 'nilmrun', packages = [ 'nilmrun',
'nilmrun.scripts', 'nilmrun.scripts',
], ],

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
import nose import nose
import os import os

View File

@ -15,14 +15,15 @@ import distutils.version
import os import os
import sys import sys
import threading import threading
import cStringIO import io
import simplejson as json import json
import unittest import unittest
import warnings import warnings
import time import time
import re import re
import urllib2 import urllib.request, urllib.error, urllib.parse
from urllib2 import urlopen, HTTPError from urllib.request import urlopen
from urllib.error import HTTPError
import requests import requests
import pprint import pprint
import textwrap import textwrap
@ -128,10 +129,27 @@ class TestClient(object):
def _run_testfilter(self, client, args): def _run_testfilter(self, client, args):
code = textwrap.dedent(""" code = textwrap.dedent("""
import nilmrun.testfilter from nilmdb.utils.printf import *
import simplejson as json import time
import signal
import json
import sys import sys
nilmrun.testfilter.test(json.loads(sys.argv[1])) # This is just for testing the process management.
def test(n):
n = int(n)
if n < 0: # raise an exception
raise Exception("test exception")
if n == 0: # ignore SIGTERM and count to 100
n = 100
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for x in range(n):
s = sprintf("dummy %d\\n", x)
if x & 1:
sys.stdout.write(s)
else:
sys.stderr.write(s)
time.sleep(0.1)
test(json.loads(sys.argv[1]))
""") """)
jsonargs = json.dumps(args) jsonargs = json.dumps(args)
return client.post("run/code", { "code": code, "args": [ jsonargs ] }) return client.post("run/code", { "code": code, "args": [ jsonargs ] })
@ -269,9 +287,9 @@ class TestClient(object):
# basic code snippet # basic code snippet
code = textwrap.dedent(""" code = textwrap.dedent("""
print 'hello' print('hello')
def foo(arg): def foo(arg):
print 'world' print('world')
""") """)
status = do(code, [], False) status = do(code, [], False)
eq_("hello\n", status["log"]) eq_("hello\n", status["log"])
@ -280,7 +298,7 @@ class TestClient(object):
# compile error # compile error
code = textwrap.dedent(""" code = textwrap.dedent("""
def foo(arg: def foo(arg:
print 'hello' print('hello')
""") """)
status = do(code, [], False) status = do(code, [], False)
in_("SyntaxError", status["log"]) in_("SyntaxError", status["log"])
@ -305,7 +323,7 @@ class TestClient(object):
# argument handling (strings come in as unicode) # argument handling (strings come in as unicode)
code = textwrap.dedent(""" code = textwrap.dedent("""
import sys import sys
print sys.argv[1].encode('ascii'), sys.argv[2] print(sys.argv[1], sys.argv[2])
sys.exit(0) # also test raising SystemExit sys.exit(0) # also test raising SystemExit
""") """)
with assert_raises(ClientError) as e: with assert_raises(ClientError) as e:
@ -318,9 +336,9 @@ class TestClient(object):
# try killing a long-running process # try killing a long-running process
code = textwrap.dedent(""" code = textwrap.dedent("""
import time import time
print 'hello' print('hello')
time.sleep(60) time.sleep(60)
print 'world' print('world')
""") """)
status = do(code, [], True) status = do(code, [], True)
eq_(status["log"], "hello\n") eq_(status["log"], "hello\n")
@ -329,7 +347,7 @@ class TestClient(object):
# default arguments are empty # default arguments are empty
code = textwrap.dedent(""" code = textwrap.dedent("""
import sys import sys
print 'args:', len(sys.argv[1:]) print('args:', len(sys.argv[1:]))
""") """)
status = do(code, None, False) status = do(code, None, False)
eq_(status["log"], "args: 0\n") eq_(status["log"], "args: 0\n")
@ -352,8 +370,10 @@ class TestClient(object):
# start some processes # start some processes
a = client.post("run/command", { "argv": ["sleep","60"] } ) a = client.post("run/command", { "argv": ["sleep","60"] } )
b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } ) b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } )
c = client.post("run/command", { "argv": ["sh","-c","burnP5;true"] } ) c = client.post("run/command", { "argv": [
d = client.post("run/command", { "argv": ["burnP5" ] } ) "sh","-c","dd if=/dev/zero of=/dev/null;true"] } )
d = client.post("run/command", { "argv": [
"dd", "if=/dev/zero", "of=/dev/null" ] } )
info = client.get("process/info") info = client.get("process/info")
eq_(info["pids"][a]["procs"], 1) eq_(info["pids"][a]["procs"], 1)
@ -365,9 +385,13 @@ class TestClient(object):
lt_(20, info["pids"][c]["cpu_percent"]) lt_(20, info["pids"][c]["cpu_percent"])
lt_(80, info["system"]["cpu_percent"]) lt_(80, info["system"]["cpu_percent"])
time.sleep(2) for x in range(10):
time.sleep(1)
info = client.get("process/info") info = client.get("process/info")
eq_(info["pids"][b]["procs"], 0) if info["pids"][b]["procs"] != 2:
break
else:
raise Exception("process B didn't die: " + str(info["pids"][b]))
# kill all processes # kill all processes
for pid in client.get("process/list"): for pid in client.get("process/list"):
@ -377,19 +401,20 @@ class TestClient(object):
client = HTTPClient(baseurl = testurl, post_json = True) client = HTTPClient(baseurl = testurl, post_json = True)
eq_(client.get("process/list"), []) eq_(client.get("process/list"), [])
def verify(cmd, result): def verify(cmd, result):
pid = client.post("run/command", { "argv": [ "sh", "-c", cmd ] }) pid = client.post("run/command", { "argv": [
"/bin/bash", "-c", cmd ] })
eq_(client.get("process/list"), [pid]) eq_(client.get("process/list"), [pid])
status = self.wait_end(client, pid) status = self.wait_end(client, pid)
eq_(result, status["log"]) eq_(result, status["log"])
# Unicode should work # Unicode should work
verify("echo -n hello", "hello") verify("echo -n hello", "hello")
verify(u"echo -n ☠", u"") verify("echo -n ☠", "")
verify("echo -ne \\\\xe2\\\\x98\\\\xa0", u"") verify("echo -ne \\\\xe2\\\\x98\\\\xa0", "")
# Programs that spit out invalid UTF-8 should get replacement # Programs that spit out invalid UTF-8 should get replacement
# markers # markers
verify("echo -ne \\\\xae", u"\ufffd") verify("echo -ne \\\\xae", "\ufffd")
def test_client_11_atexit(self): def test_client_11_atexit(self):
# Leave a directory and running process behind, for the atexit # Leave a directory and running process behind, for the atexit

View File

@ -1,7 +1,7 @@
# Just some helpers for test functions # Just some helpers for test functions
def myrepr(x): def myrepr(x):
if isinstance(x, basestring): if isinstance(x, str):
return '"' + x + '"' return '"' + x + '"'
else: else:
return repr(x) return repr(x)

File diff suppressed because it is too large Load Diff