Compare commits

...

24 Commits

Author SHA1 Message Date
eae6dd623f Freeze requirements 2020-08-06 18:13:01 -04:00
09a9ed9734 Fix #! at top of shell scripts for py3 and venvs 2020-08-05 17:01:47 -04:00
079a2b5192 Fix nilmrun scripts for python 3 2020-08-05 16:59:56 -04:00
e7f52a4013 Update versioneer 2020-08-05 16:42:46 -04:00
c36b9b97e0 Add missing MANIFEST.in, update .gitignore 2020-08-03 23:48:07 -04:00
549a27e66c Add flake8 tests and clean up warnings 2020-08-03 23:46:34 -04:00
cd68389e9a Update for Python 3 and psutil >= 2. 2020-08-03 23:08:52 -04:00
6faf563bda Remove nilmrun.testfilter
This module was only used to assist testing, but wasn't actually
available in a test environment where no copies of nilmrun have been
installed.  Just code the contents directly into the filters that the
tests are running.
2020-08-03 23:05:32 -04:00
58fd9d1559 Run cherrypy always in "embedded" mode
Non-embedded mode is not used in the test suite or wsgi server;
it was an option in the standalong nilmdb-server script, but it's
really not necessary, and removing it gets rid of some untested
code.
2020-08-03 16:57:17 -04:00
2bc939d42d CherryPy bug 1200 is no longer an issue 2020-08-03 16:55:23 -04:00
fe36722684 Use built-in json module rather than external simplejson 2020-08-03 16:54:47 -04:00
24740a838e Run 2to3 over all code 2020-08-03 16:54:33 -04:00
d332fa1e0f Fix inconsistent indentation 2020-08-03 16:52:02 -04:00
cbc7a7dd55 Start Python 3 conversion 2020-08-03 16:51:01 -04:00
38c3e67cf9 Fix long lines 2013-08-08 17:47:06 -04:00
7f05a0fb62 Switch to bash for Unicode tests 2013-08-07 12:42:05 -04:00
81c2ad07d4 More robust checking in 09_info test 2013-08-07 12:32:21 -04:00
3588e843ac Replace burnP5 with dd 2013-08-07 12:31:43 -04:00
9309fd9b57 Change -V option to -v everywhere 2013-08-06 21:33:59 -04:00
21bd1bd050 Always print header, even if no processes 2013-08-06 14:56:10 -04:00
cafdfce4f0 Add nilmrun-ps, -kill, and -run commands 2013-08-06 14:38:43 -04:00
477c27a4e6 Clean up temp dirs and processes at exit 2013-07-30 20:15:01 -04:00
bed26e099b Put each code in its own dir; save args too 2013-07-30 20:14:48 -04:00
9224566f9b More robust process killing 2013-07-30 20:13:30 -04:00
22 changed files with 2623 additions and 893 deletions

2
.gitattributes vendored
View File

@ -1 +1 @@
src/_version.py export-subst nilmrun/_version.py export-subst

2
.gitignore vendored
View File

@ -3,8 +3,8 @@ build/
*.pyc *.pyc
dist/ dist/
nilmrun.egg-info/ nilmrun.egg-info/
.eggs/
# This gets generated as needed by setup.py # This gets generated as needed by setup.py
MANIFEST.in
MANIFEST MANIFEST

8
MANIFEST.in Normal file
View File

@ -0,0 +1,8 @@
# Root
include README.md
include setup.py
include versioneer.py
include Makefile
# Version
include nilmrun/_version.py

View File

@ -1,48 +1,49 @@
# By default, run the tests. # By default, run the tests.
all: test all: test
test2:
nilmrun/trainola.py data.js
version: version:
python setup.py version python3 setup.py version
build: build:
python setup.py build_ext --inplace python3 setup.py build_ext --inplace
dist: sdist dist: sdist
sdist: sdist:
python setup.py sdist python3 setup.py sdist
install: install:
python setup.py install python3 setup.py install
develop: develop:
python setup.py develop python3 setup.py develop
docs: docs:
make -C docs make -C docs
ctrl: flake
flake:
flake8 nilmrun
lint: lint:
pylint --rcfile=.pylintrc nilmdb pylint3 --rcfile=setup.cfg nilmrun
test: test:
ifeq ($(INSIDE_EMACS), t) ifneq ($(INSIDE_EMACS),)
# Use the slightly more flexible script # Use the slightly more flexible script
python setup.py build_ext --inplace python3 setup.py build_ext --inplace
python tests/runtests.py python3 tests/runtests.py
else else
# Let setup.py check dependencies, build stuff, and run the test # Let setup.py check dependencies, build stuff, and run the test
python setup.py nosetests python3 setup.py nosetests
endif endif
clean:: clean::
find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
rm -f .coverage rm -f .coverage
find . -name '*pyc' | xargs rm -f rm -rf nilmrun.egg-info/ build/
rm -rf nilmtools.egg-info/ build/ MANIFEST.in
make -C docs clean make -C docs clean
gitclean:: gitclean::
git clean -dXf git clean -dXf
.PHONY: all version dist sdist install docs lint test clean gitclean .PHONY: all version dist sdist install docs test
.PHONY: ctrl lint flake clean gitclean

32
README.md Normal file
View File

@ -0,0 +1,32 @@
# nilmrun: Run NilmDB filters
by Jim Paris <jim@jtan.com>
## Prerequisites:
# Runtime and build environments
sudo apt-get install python3
# Create a new Python virtual environment to isolate deps.
python3 -m venv ../venv
source ../venv/bin/activate # run "deactivate" to leave
# Install all Python dependencies
pip3 install -r requirements.txt
## Install:
Install it into the virtual environment
python3 setup.py install
If you want to instead install it system-wide, you will also need to
install the requirements system-wide:
sudo pip3 install -r requirements.txt
sudo python3 setup.py install
## Usage:
nilmrun-server --help
See docs/wsgi.md for info on setting up a WSGI application in Apache.

View File

@ -1,20 +0,0 @@
nilmrun: Run NilmDB filters
by Jim Paris <jim@jtan.com>
Prerequisites:
# Runtime and build environments
sudo apt-get install python2.7 python-setuptools
# Plus nilmdb and its dependencies
nilmdb (1.8.3+)
Install:
python setup.py install
Usage:
nilmrun-server --help
See docs/wsgi.md for info on setting up a WSGI application in Apache.

View File

@ -1,5 +1,3 @@
import nilmrun.processmanager
from ._version import get_versions from ._version import get_versions
__version__ = get_versions()['version'] __version__ = get_versions()['version']
del get_versions del get_versions

View File

@ -1,197 +1,520 @@
IN_LONG_VERSION_PY = True
# This file helps to compute a version number in source trees obtained from # This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag # git-archive tarball (such as those provided by githubs download-from-tag
# feature). Distribution tarballs (build by setup.py sdist) and build # feature). Distribution tarballs (built by setup.py sdist) and build
# directories (produced by setup.py build) will contain a much shorter file # directories (produced by setup.py build) will contain a much shorter file
# that just contains the computed version number. # that just contains the computed version number.
# This file is released into the public domain. Generated by # This file is released into the public domain. Generated by
# versioneer-0.7+ (https://github.com/warner/python-versioneer) # versioneer-0.18 (https://github.com/warner/python-versioneer)
# these strings will be replaced by git during git-archive
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
"""Git implementation of _version.py."""
import errno
import os
import re
import subprocess import subprocess
import sys import sys
def run_command(args, cwd=None, verbose=False):
try: def get_keywords():
# remember shell=False, so use git.cmd on windows, not just git """Get the keywords needed to look up the version information."""
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) # these strings will be replaced by git during git-archive.
except EnvironmentError: # setup.py/versioneer.py will grep for the variable names, so they must
e = sys.exc_info()[1] # each be defined on a line of their own. _version.py will just call
# get_keywords().
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
git_date = "$Format:%ci$"
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
return keywords
class VersioneerConfig:
"""Container for Versioneer configuration parameters."""
def get_config():
"""Create, populate and return the VersioneerConfig() object."""
# these strings are filled in when 'setup.py versioneer' creates
# _version.py
cfg = VersioneerConfig()
cfg.VCS = "git"
cfg.style = "pep440"
cfg.tag_prefix = "nilmrun-"
cfg.parentdir_prefix = "nilmrun-"
cfg.versionfile_source = "nilmrun/_version.py"
cfg.verbose = False
return cfg
class NotThisMethod(Exception):
"""Exception raised if a method is not valid for the current scenario."""
LONG_VERSION_PY = {}
HANDLERS = {}
def register_vcs_handler(vcs, method): # decorator
"""Decorator to mark a method as the handler for a particular VCS."""
def decorate(f):
"""Store f in HANDLERS[vcs][method]."""
if vcs not in HANDLERS:
HANDLERS[vcs] = {}
HANDLERS[vcs][method] = f
return f
return decorate
def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
env=None):
"""Call the given command(s)."""
assert isinstance(commands, list)
p = None
for c in commands:
try:
dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git
p = subprocess.Popen([c] + args, cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
break
except EnvironmentError:
e = sys.exc_info()[1]
if e.errno == errno.ENOENT:
continue
if verbose:
print("unable to run %s" % dispcmd)
print(e)
return None, None
else:
if verbose: if verbose:
print("unable to run %s" % args[0]) print("unable to find command, tried %s" % (commands,))
print(e) return None, None
return None
stdout = p.communicate()[0].strip() stdout = p.communicate()[0].strip()
if sys.version >= '3': if sys.version_info[0] >= 3:
stdout = stdout.decode() stdout = stdout.decode()
if p.returncode != 0: if p.returncode != 0:
if verbose: if verbose:
print("unable to run %s (error)" % args[0]) print("unable to run %s (error)" % dispcmd)
return None print("stdout was %s" % stdout)
return stdout return None, p.returncode
return stdout, p.returncode
import sys def versions_from_parentdir(parentdir_prefix, root, verbose):
import re """Try to determine the version from the parent directory name.
import os.path
def get_expanded_variables(versionfile_source): Source tarballs conventionally unpack into a directory that includes both
the project name and a version string. We will also support searching up
two directory levels for an appropriately named parent directory
"""
rootdirs = []
for i in range(3):
dirname = os.path.basename(root)
if dirname.startswith(parentdir_prefix):
return {"version": dirname[len(parentdir_prefix):],
"full-revisionid": None,
"dirty": False, "error": None, "date": None}
else:
rootdirs.append(root)
root = os.path.dirname(root) # up a level
if verbose:
print("Tried directories %s but none started with prefix %s" %
(str(rootdirs), parentdir_prefix))
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
@register_vcs_handler("git", "get_keywords")
def git_get_keywords(versionfile_abs):
"""Extract version information from the given file."""
# the code embedded in _version.py can just fetch the value of these # the code embedded in _version.py can just fetch the value of these
# variables. When used from setup.py, we don't want to import # keywords. When used from setup.py, we don't want to import _version.py,
# _version.py, so we do it with a regexp instead. This function is not # so we do it with a regexp instead. This function is not used from
# used from _version.py. # _version.py.
variables = {} keywords = {}
try: try:
for line in open(versionfile_source,"r").readlines(): f = open(versionfile_abs, "r")
for line in f.readlines():
if line.strip().startswith("git_refnames ="): if line.strip().startswith("git_refnames ="):
mo = re.search(r'=\s*"(.*)"', line) mo = re.search(r'=\s*"(.*)"', line)
if mo: if mo:
variables["refnames"] = mo.group(1) keywords["refnames"] = mo.group(1)
if line.strip().startswith("git_full ="): if line.strip().startswith("git_full ="):
mo = re.search(r'=\s*"(.*)"', line) mo = re.search(r'=\s*"(.*)"', line)
if mo: if mo:
variables["full"] = mo.group(1) keywords["full"] = mo.group(1)
if line.strip().startswith("git_date ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["date"] = mo.group(1)
f.close()
except EnvironmentError: except EnvironmentError:
pass pass
return variables return keywords
def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
refnames = variables["refnames"].strip() @register_vcs_handler("git", "keywords")
def git_versions_from_keywords(keywords, tag_prefix, verbose):
"""Get version information from git keywords."""
if not keywords:
raise NotThisMethod("no keywords at all, weird")
date = keywords.get("date")
if date is not None:
# git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
# datestamp. However we prefer "%ci" (which expands to an "ISO-8601
# -like" string, which we must then edit to make compliant), because
# it's been around since git-1.5.3, and it's too difficult to
# discover which version we're using, or to work around using an
# older one.
date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
refnames = keywords["refnames"].strip()
if refnames.startswith("$Format"): if refnames.startswith("$Format"):
if verbose: if verbose:
print("variables are unexpanded, not using") print("keywords are unexpanded, not using")
return {} # unexpanded, so not in an unpacked git-archive tarball raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
refs = set([r.strip() for r in refnames.strip("()").split(",")]) refs = set([r.strip() for r in refnames.strip("()").split(",")])
for ref in list(refs): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
if not re.search(r'\d', ref): # just "foo-1.0". If we see a "tag: " prefix, prefer those.
if verbose: TAG = "tag: "
print("discarding '%s', no digits" % ref) tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
refs.discard(ref) if not tags:
# Assume all version tags have a digit. git's %d expansion # Either we're using git < 1.8.3, or there really are no tags. We use
# behaves like git log --decorate=short and strips out the # a heuristic: assume all version tags have a digit. The old git %d
# refs/heads/ and refs/tags/ prefixes that would let us # expansion behaves like git log --decorate=short and strips out the
# distinguish between branches and tags. By ignoring refnames # refs/heads/ and refs/tags/ prefixes that would let us distinguish
# without digits, we filter out many common branch names like # between branches and tags. By ignoring refnames without digits, we
# "release" and "stabilization", as well as "HEAD" and "master". # filter out many common branch names like "release" and
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose:
print("discarding '%s', no digits" % ",".join(refs - tags))
if verbose: if verbose:
print("remaining refs: %s" % ",".join(sorted(refs))) print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(refs): for ref in sorted(tags):
# sorting will prefer e.g. "2.0" over "2.0rc1" # sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix): if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):] r = ref[len(tag_prefix):]
if verbose: if verbose:
print("picking %s" % r) print("picking %s" % r)
return { "version": r, return {"version": r,
"full": variables["full"].strip() } "full-revisionid": keywords["full"].strip(),
# no suitable tags, so we use the full revision id "dirty": False, "error": None,
"date": date}
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose: if verbose:
print("no suitable tags, using full revision id") print("no suitable tags, using unknown + full revision id")
return { "version": variables["full"].strip(), return {"version": "0+unknown",
"full": variables["full"].strip() } "full-revisionid": keywords["full"].strip(),
"dirty": False, "error": "no suitable tags", "date": None}
def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
# this runs 'git' from the root of the source tree. That either means @register_vcs_handler("git", "pieces_from_vcs")
# someone ran a setup.py command (and this code is in versioneer.py, so def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
# IN_LONG_VERSION_PY=False, thus the containing directory is the root of """Get version from 'git describe' in the root of the source tree.
# the source tree), or someone ran a project-specific entry point (and
# this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the This only gets called if the git-archive 'subst' keywords were *not*
# containing directory is somewhere deeper in the source tree). This only expanded, and _version.py hasn't already been rewritten with a short
# gets called if the git-archive 'subst' variables were *not* expanded, version string, meaning we're inside a checked out source tree.
# and _version.py hasn't already been rewritten with a short version """
# string, meaning we're inside a checked out source tree. GITS = ["git"]
if sys.platform == "win32":
GITS = ["git.cmd", "git.exe"]
out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
hide_stderr=True)
if rc != 0:
if verbose:
print("Directory %s not under git control" % root)
raise NotThisMethod("'git rev-parse --git-dir' returned error")
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
# if there isn't one, this yields HEX[-dirty] (no NUM)
describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
"--always", "--long",
"--match", "%s*" % tag_prefix],
cwd=root)
# --long was added in git-1.5.5
if describe_out is None:
raise NotThisMethod("'git describe' failed")
describe_out = describe_out.strip()
full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
if full_out is None:
raise NotThisMethod("'git rev-parse' failed")
full_out = full_out.strip()
pieces = {}
pieces["long"] = full_out
pieces["short"] = full_out[:7] # maybe improved later
pieces["error"] = None
# parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
# TAG might have hyphens.
git_describe = describe_out
# look for -dirty suffix
dirty = git_describe.endswith("-dirty")
pieces["dirty"] = dirty
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
# now we have TAG-NUM-gHEX or HEX
if "-" in git_describe:
# TAG-NUM-gHEX
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
if not mo:
# unparseable. Maybe git-describe is misbehaving?
pieces["error"] = ("unable to parse git-describe output: '%s'"
% describe_out)
return pieces
# tag
full_tag = mo.group(1)
if not full_tag.startswith(tag_prefix):
if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'"
print(fmt % (full_tag, tag_prefix))
pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
% (full_tag, tag_prefix))
return pieces
pieces["closest-tag"] = full_tag[len(tag_prefix):]
# distance: number of commits since tag
pieces["distance"] = int(mo.group(2))
# commit: short hex revision ID
pieces["short"] = mo.group(3)
else:
# HEX: no tags
pieces["closest-tag"] = None
count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
cwd=root)
pieces["distance"] = int(count_out) # total number of commits
# commit date: see ISO-8601 comment in git_versions_from_keywords()
date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
cwd=root)[0].strip()
pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
return pieces
def plus_or_dot(pieces):
"""Return a + if we don't already have one, else return a ."""
if "+" in pieces.get("closest-tag", ""):
return "."
return "+"
def render_pep440(pieces):
"""Build up version string, with post-release "local version identifier".
Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
Exceptions:
1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += plus_or_dot(pieces)
rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
else:
# exception #1
rendered = "0+untagged.%d.g%s" % (pieces["distance"],
pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
return rendered
def render_pep440_pre(pieces):
"""TAG[.post.devDISTANCE] -- No -dirty.
Exceptions:
1: no tags. 0.post.devDISTANCE
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"]:
rendered += ".post.dev%d" % pieces["distance"]
else:
# exception #1
rendered = "0.post.dev%d" % pieces["distance"]
return rendered
def render_pep440_post(pieces):
"""TAG[.postDISTANCE[.dev0]+gHEX] .
The ".dev0" means dirty. Note that .dev0 sorts backwards
(a dirty tree will appear "older" than the corresponding clean one),
but you shouldn't be releasing software with -dirty anyways.
Exceptions:
1: no tags. 0.postDISTANCE[.dev0]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += ".post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
rendered += plus_or_dot(pieces)
rendered += "g%s" % pieces["short"]
else:
# exception #1
rendered = "0.post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
rendered += "+g%s" % pieces["short"]
return rendered
def render_pep440_old(pieces):
"""TAG[.postDISTANCE[.dev0]] .
The ".dev0" means dirty.
Eexceptions:
1: no tags. 0.postDISTANCE[.dev0]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += ".post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
else:
# exception #1
rendered = "0.post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
return rendered
def render_git_describe(pieces):
"""TAG[-DISTANCE-gHEX][-dirty].
Like 'git describe --tags --dirty --always'.
Exceptions:
1: no tags. HEX[-dirty] (note: no 'g' prefix)
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"]:
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
else:
# exception #1
rendered = pieces["short"]
if pieces["dirty"]:
rendered += "-dirty"
return rendered
def render_git_describe_long(pieces):
"""TAG-DISTANCE-gHEX[-dirty].
Like 'git describe --tags --dirty --always -long'.
The distance/hash is unconditional.
Exceptions:
1: no tags. HEX[-dirty] (note: no 'g' prefix)
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
else:
# exception #1
rendered = pieces["short"]
if pieces["dirty"]:
rendered += "-dirty"
return rendered
def render(pieces, style):
"""Render the given version pieces into the requested style."""
if pieces["error"]:
return {"version": "unknown",
"full-revisionid": pieces.get("long"),
"dirty": None,
"error": pieces["error"],
"date": None}
if not style or style == "default":
style = "pep440" # the default
if style == "pep440":
rendered = render_pep440(pieces)
elif style == "pep440-pre":
rendered = render_pep440_pre(pieces)
elif style == "pep440-post":
rendered = render_pep440_post(pieces)
elif style == "pep440-old":
rendered = render_pep440_old(pieces)
elif style == "git-describe":
rendered = render_git_describe(pieces)
elif style == "git-describe-long":
rendered = render_git_describe_long(pieces)
else:
raise ValueError("unknown style '%s'" % style)
return {"version": rendered, "full-revisionid": pieces["long"],
"dirty": pieces["dirty"], "error": None,
"date": pieces.get("date")}
def get_versions():
"""Get version information or return default if unable to do so."""
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
# __file__, we can work backwards from there to the root. Some
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
# case we can only use expanded keywords.
cfg = get_config()
verbose = cfg.verbose
try: try:
here = os.path.abspath(__file__) return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
except NameError: verbose)
# some py2exe/bbfreeze/non-CPython implementations don't do __file__ except NotThisMethod:
return {} # not always correct pass
# versionfile_source is the relative path from the top of the source tree try:
# (where the .git directory might live) to this file. Invert this to find root = os.path.realpath(__file__)
# the root from __file__.
root = here
if IN_LONG_VERSION_PY:
for i in range(len(versionfile_source.split("/"))):
root = os.path.dirname(root)
else:
root = os.path.dirname(here)
if not os.path.exists(os.path.join(root, ".git")):
if verbose:
print("no .git in %s" % root)
return {}
GIT = "git"
if sys.platform == "win32":
GIT = "git.cmd"
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
cwd=root)
if stdout is None:
return {}
if not stdout.startswith(tag_prefix):
if verbose:
print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
return {}
tag = stdout[len(tag_prefix):]
stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
if stdout is None:
return {}
full = stdout.strip()
if tag.endswith("-dirty"):
full += "-dirty"
return {"version": tag, "full": full}
def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
if IN_LONG_VERSION_PY:
# We're running from _version.py. If it's from a source tree
# (execute-in-place), we can work upwards to find the root of the
# tree, and then check the parent directory for a version string. If
# it's in an installed application, there's no hope.
try:
here = os.path.abspath(__file__)
except NameError:
# py2exe/bbfreeze/non-CPython don't have __file__
return {} # without __file__, we have no hope
# versionfile_source is the relative path from the top of the source # versionfile_source is the relative path from the top of the source
# tree to _version.py. Invert this to find the root from __file__. # tree (where the .git directory might live) to this file. Invert
root = here # this to find the root from __file__.
for i in range(len(versionfile_source.split("/"))): for i in cfg.versionfile_source.split('/'):
root = os.path.dirname(root) root = os.path.dirname(root)
else: except NameError:
# we're running from versioneer.py, which means we're running from return {"version": "0+unknown", "full-revisionid": None,
# the setup.py in a source tree. sys.argv[0] is setup.py in the root. "dirty": None,
here = os.path.abspath(sys.argv[0]) "error": "unable to find root of source tree",
root = os.path.dirname(here) "date": None}
# Source tarballs conventionally unpack into a directory that includes try:
# both the project name and a version string. pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
dirname = os.path.basename(root) return render(pieces, cfg.style)
if not dirname.startswith(parentdir_prefix): except NotThisMethod:
if verbose: pass
print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
(root, dirname, parentdir_prefix))
return None
return {"version": dirname[len(parentdir_prefix):], "full": ""}
tag_prefix = "nilmrun-" try:
parentdir_prefix = "nilmrun-" if cfg.parentdir_prefix:
versionfile_source = "nilmrun/_version.py" return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
except NotThisMethod:
def get_versions(default={"version": "unknown", "full": ""}, verbose=False): pass
variables = { "refnames": git_refnames, "full": git_full }
ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
if not ver:
ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
if not ver:
ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
verbose)
if not ver:
ver = default
return ver
return {"version": "0+unknown", "full-revisionid": None,
"dirty": None,
"error": "unable to compute version", "date": None}

View File

@ -1,10 +1,8 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.utils.printf import *
import threading import threading
import subprocess import subprocess
import cStringIO import io
import sys import sys
import os import os
import signal import signal
@ -15,16 +13,18 @@ import tempfile
import atexit import atexit
import shutil import shutil
class ProcessError(Exception): class ProcessError(Exception):
pass pass
class LogReceiver(object): class LogReceiver(object):
"""Spawn a thread that listens to a pipe for log messages, """Spawn a thread that listens to a pipe for log messages,
and stores them locally.""" and stores them locally."""
def __init__(self, pipe): def __init__(self, pipe):
self.pipe = pipe self.pipe = pipe
self.log = cStringIO.StringIO() self.log = io.BytesIO()
self.thread = threading.Thread(target = self.run) self.thread = threading.Thread(target=self.run)
self.thread.start() self.thread.start()
def run(self): def run(self):
@ -39,11 +39,12 @@ class LogReceiver(object):
return self.log.getvalue() return self.log.getvalue()
def clear(self): def clear(self):
self.log = cStringIO.StringIO() self.log = io.BytesIO()
class Process(object): class Process(object):
"""Spawn and manage a subprocess, and capture its output.""" """Spawn and manage a subprocess, and capture its output."""
def __init__(self, argv, tempfile = None): def __init__(self, argv, tempfile=None):
self.start_time = None self.start_time = None
# Use a pipe for communicating log data # Use a pipe for communicating log data
@ -55,9 +56,9 @@ class Process(object):
# Spawn the new process # Spawn the new process
try: try:
self._process = subprocess.Popen(args = argv, stdin = nullfd, self._process = subprocess.Popen(args=argv, stdin=nullfd,
stdout = wpipe, stderr = wpipe, stdout=wpipe, stderr=wpipe,
close_fds = True, cwd = "/tmp") close_fds=True, cwd="/tmp")
except (OSError, TypeError) as e: except (OSError, TypeError) as e:
raise ProcessError(str(e)) raise ProcessError(str(e))
finally: finally:
@ -69,7 +70,7 @@ class Process(object):
self.start_time = time.time() self.start_time = time.time()
self.pid = str(uuid.uuid1(self._process.pid or 0)) self.pid = str(uuid.uuid1(self._process.pid or 0))
def _join(self, timeout = 1.0): def _join(self, timeout=1.0):
start = time.time() start = time.time()
while True: while True:
if self._process.poll() is not None: if self._process.poll() is not None:
@ -78,7 +79,7 @@ class Process(object):
return False return False
time.sleep(0.1) time.sleep(0.1)
def terminate(self, timeout = 1.0): def terminate(self, timeout=1.0):
"""Terminate a process, and all of its children that are in the same """Terminate a process, and all of its children that are in the same
process group.""" process group."""
try: try:
@ -89,18 +90,24 @@ class Process(object):
def getpgid(pid): def getpgid(pid):
try: try:
return os.getpgid(pid) return os.getpgid(pid)
except OSError: # pragma: no cover except OSError: # pragma: no cover
return None return None
def kill(pid, sig):
try:
return os.kill(pid, sig)
except OSError: # pragma: no cover
return
# Find all children # Find all children
group = getpgid(self._process.pid) group = getpgid(self._process.pid)
main = psutil.Process(self._process.pid) main = psutil.Process(self._process.pid)
allproc = [ main ] + main.get_children(recursive = True) allproc = [main] + main.children(recursive=True)
# Kill with SIGTERM, if they're still in this process group # Kill with SIGTERM, if they're still in this process group
for proc in allproc: for proc in allproc:
if getpgid(proc.pid) == group: if getpgid(proc.pid) == group:
os.kill(proc.pid, signal.SIGTERM) kill(proc.pid, signal.SIGTERM)
# Wait for it to die again # Wait for it to die again
if self._join(timeout): if self._join(timeout):
@ -109,11 +116,11 @@ class Process(object):
# One more try with SIGKILL # One more try with SIGKILL
for proc in allproc: for proc in allproc:
if getpgid(proc.pid) == group: if getpgid(proc.pid) == group:
os.kill(proc.pid, signal.SIGKILL) kill(proc.pid, signal.SIGKILL)
# See if it worked # See if it worked
return self._join(timeout) return self._join(timeout)
except psutil.Error: # pragma: no cover (race condition) except psutil.Error: # pragma: no cover (race condition)
return True return True
def clear_log(self): def clear_log(self):
@ -136,22 +143,22 @@ class Process(object):
Call .get_info() about a second later.""" Call .get_info() about a second later."""
try: try:
main = psutil.Process(self._process.pid) main = psutil.Process(self._process.pid)
self._process_list = [ main ] + main.get_children(recursive = True) self._process_list = [main] + main.children(recursive=True)
for proc in self._process_list: for proc in self._process_list:
proc.get_cpu_percent(0) proc.cpu_percent(0)
except psutil.Error: # pragma: no cover (race condition) except psutil.Error: # pragma: no cover (race condition)
self._process_list = [ ] self._process_list = []
@staticmethod @staticmethod
def get_empty_info(): def get_empty_info():
return { "cpu_percent": 0, return {"cpu_percent": 0,
"cpu_user": 0, "cpu_user": 0,
"cpu_sys": 0, "cpu_sys": 0,
"mem_phys": 0, "mem_phys": 0,
"mem_virt": 0, "mem_virt": 0,
"io_read": 0, "io_read": 0,
"io_write": 0, "io_write": 0,
"procs": 0 } "procs": 0}
def get_info(self): def get_info(self):
"""Return a dictionary with info about the process CPU and memory """Return a dictionary with info about the process CPU and memory
@ -159,14 +166,14 @@ class Process(object):
d = self.get_empty_info() d = self.get_empty_info()
for proc in self._process_list: for proc in self._process_list:
try: try:
d["cpu_percent"] += proc.get_cpu_percent(0) d["cpu_percent"] += proc.cpu_percent(0)
cpuinfo = proc.get_cpu_times() cpuinfo = proc.cpu_times()
d["cpu_user"] += cpuinfo.user d["cpu_user"] += cpuinfo.user
d["cpu_sys"] += cpuinfo.system d["cpu_sys"] += cpuinfo.system
meminfo = proc.get_memory_info() meminfo = proc.memory_info()
d["mem_phys"] += meminfo.rss d["mem_phys"] += meminfo.rss
d["mem_virt"] += meminfo.vms d["mem_virt"] += meminfo.vms
ioinfo = proc.get_io_counters() ioinfo = proc.io_counters()
d["io_read"] += ioinfo.read_bytes d["io_read"] += ioinfo.read_bytes
d["io_write"] += ioinfo.write_bytes d["io_write"] += ioinfo.write_bytes
d["procs"] += 1 d["procs"] += 1
@ -174,16 +181,35 @@ class Process(object):
pass pass
return d return d
class ProcessManager(object): class ProcessManager(object):
"""Track and manage a collection of Process objects""" """Track and manage a collection of Process objects"""
def __init__(self): def __init__(self):
self.processes = {} self.processes = {}
self.tmpfiles = {} self.tmpdirs = {}
self.tmpdir = tempfile.mkdtemp(prefix = "nilmrun-usercode-") atexit.register(self._atexit)
atexit.register(shutil.rmtree, self.tmpdir)
def _cleanup_tmpdir(self, pid):
if pid in self.tmpdirs:
try:
shutil.rmtree(self.tmpdirs[pid])
except OSError: # pragma: no cover
pass
del self.tmpdirs[pid]
def _atexit(self):
# Kill remaining processes, remove their dirs
for pid in list(self.processes.keys()):
try:
self.processes[pid].terminate()
del self.processes[pid]
shutil.rmtree(self.tmpdirs[pid])
del self.tmpdirs[pid]
except Exception: # pragma: no cover
pass
def __iter__(self): def __iter__(self):
return iter(self.processes.keys()) return iter(list(self.processes.keys()))
def __getitem__(self, key): def __getitem__(self, key):
return self.processes[key] return self.processes[key]
@ -193,15 +219,33 @@ class ProcessManager(object):
executed. The arguments, which must be strings, will be executed. The arguments, which must be strings, will be
accessible in the code as sys.argv[1:].""" accessible in the code as sys.argv[1:]."""
# The easiest way to do this, by far, is to just write the # The easiest way to do this, by far, is to just write the
# code to a file. # code to a file. Make a directory to put it in.
(fd, path) = tempfile.mkstemp(prefix = "nilmrun-usercode-", tmpdir = tempfile.mkdtemp(prefix="nilmrun-usercode-")
suffix = ".py", dir=self.tmpdir) try:
with os.fdopen(fd, 'w') as f: # Write the code
f.write(code) codepath = os.path.join(tmpdir, "usercode.py")
argv = [ sys.executable, "-B", "-s", "-u", path ] + args with open(codepath, "w") as f:
pid = self.run_command(argv) f.write(code)
self.tmpfiles[pid] = path # Save the args too, for debugging purposes
return pid with open(os.path.join(tmpdir, "args.txt"), "w") as f:
f.write(repr(args))
# Run the code
argv = [sys.executable, "-B", "-s", "-u", codepath] + args
pid = self.run_command(argv)
# Save the temp dir
self.tmpdirs[pid] = tmpdir
tmpdir = None # Don't need to remove it anymore
return pid
finally:
# Clean up tempdir if we didn't finish
if tmpdir is not None:
try:
shutil.rmtree(tmpdir)
except OSError: # pragma: no cover
pass
def run_command(self, argv): def run_command(self, argv):
"""Execute a command line program""" """Execute a command line program"""
@ -213,38 +257,32 @@ class ProcessManager(object):
return self.processes[pid].terminate() return self.processes[pid].terminate()
def remove(self, pid): def remove(self, pid):
if pid in self.tmpfiles: self._cleanup_tmpdir(pid)
try:
os.unlink(self.tmpfiles[pid])
except OSError: # pragma: no cover
pass
del self.tmpfiles[pid]
del self.processes[pid] del self.processes[pid]
def get_info(self): def get_info(self):
"""Get info about all running PIDs""" """Get info about all running PIDs"""
info = { "total" : Process.get_empty_info(), info = {
"pids" : {}, "total": Process.get_empty_info(),
"system" : {} "pids": {},
} "system": {}
}
# Trigger CPU usage collection # Trigger CPU usage collection
for pid in self: for pid in self:
self[pid].get_info_prepare() self[pid].get_info_prepare()
psutil.cpu_percent(0, percpu = True) psutil.cpu_percent(0, percpu=True)
# Give it some time # Give it some time
time.sleep(1) time.sleep(1)
# Retrieve info for system # Retrieve info for system
info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True)) info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True))
info["system"]["cpu_max"] = 100.0 * psutil.NUM_CPUS info["system"]["cpu_max"] = 100.0 * psutil.cpu_count()
info["system"]["procs"] = len(psutil.get_pid_list()) info["system"]["procs"] = len(psutil.pids())
# psutil > 0.6.0's psutil.virtual_memory() would be better here, meminfo = psutil.virtual_memory()
# but this should give the same info.
meminfo = psutil.phymem_usage()
info["system"]["mem_total"] = meminfo.total info["system"]["mem_total"] = meminfo.total
info["system"]["mem_used"] = int(meminfo.total * meminfo.percent / 100) info["system"]["mem_used"] = meminfo.used
# Retrieve info for each PID # Retrieve info for each PID
for pid in self: for pid in self:

View File

@ -1,19 +1,12 @@
"""CherryPy-based server for running NILM filters via HTTP""" """CherryPy-based server for running NILM filters via HTTP"""
import cherrypy import cherrypy
import sys
import os import os
import socket import socket
import simplejson as json
import traceback import traceback
import time
import nilmdb from nilmdb.utils.printf import sprintf
from nilmdb.utils.printf import *
from nilmdb.server.serverutil import ( from nilmdb.server.serverutil import (
chunked_response,
response_type,
workaround_cp_bug_1200,
exception_to_httperror, exception_to_httperror,
CORS_allow, CORS_allow,
json_to_request_params, json_to_request_params,
@ -24,11 +17,12 @@ from nilmdb.server.serverutil import (
) )
from nilmdb.utils import serializer_proxy from nilmdb.utils import serializer_proxy
import nilmrun import nilmrun
import nilmrun.testfilter import nilmrun.processmanager
# Add CORS_allow tool # Add CORS_allow tool
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow) cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
# CherryPy apps # CherryPy apps
class App(object): class App(object):
"""Root application for NILM runner""" """Root application for NILM runner"""
@ -55,6 +49,7 @@ class App(object):
def version(self): def version(self):
return nilmrun.__version__ return nilmrun.__version__
class AppProcess(object): class AppProcess(object):
def __init__(self, manager): def __init__(self, manager):
@ -76,7 +71,7 @@ class AppProcess(object):
# /process/status # /process/status
@cherrypy.expose @cherrypy.expose
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
def status(self, pid, clear = False): def status(self, pid, clear=False):
"""Return status about a process. If clear = True, also clear """Return status about a process. If clear = True, also clear
the log.""" the log."""
clear = bool_param(clear) clear = bool_param(clear)
@ -106,18 +101,19 @@ class AppProcess(object):
@cherrypy.expose @cherrypy.expose
@cherrypy.tools.json_in() @cherrypy.tools.json_in()
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
@cherrypy.tools.CORS_allow(methods = ["POST"]) @cherrypy.tools.CORS_allow(methods=["POST"])
def remove(self, pid): def remove(self, pid):
"""Remove a process from the manager, killing it if necessary.""" """Remove a process from the manager, killing it if necessary."""
if pid not in self.manager: if pid not in self.manager:
raise cherrypy.HTTPError("404 Not Found", "No such PID") raise cherrypy.HTTPError("404 Not Found", "No such PID")
if not self.manager.terminate(pid): # pragma: no cover if not self.manager.terminate(pid): # pragma: no cover
raise cherrypy.HTTPError("503 Service Unavailable", raise cherrypy.HTTPError("503 Service Unavailable",
"Failed to stop process") "Failed to stop process")
status = self.process_status(pid) status = self.process_status(pid)
self.manager.remove(pid) self.manager.remove(pid)
return status return status
class AppRun(object): class AppRun(object):
def __init__(self, manager): def __init__(self, manager):
self.manager = manager self.manager = manager
@ -127,7 +123,7 @@ class AppRun(object):
@cherrypy.tools.json_in() @cherrypy.tools.json_in()
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError) @exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods = ["POST"]) @cherrypy.tools.CORS_allow(methods=["POST"])
def command(self, argv): def command(self, argv):
"""Execute an arbitrary program on the server. argv is a """Execute an arbitrary program on the server. argv is a
list of the program and its arguments: 'argv[0]' is the program list of the program and its arguments: 'argv[0]' is the program
@ -142,8 +138,8 @@ class AppRun(object):
@cherrypy.tools.json_in() @cherrypy.tools.json_in()
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError) @exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods = ["POST"]) @cherrypy.tools.CORS_allow(methods=["POST"])
def code(self, code, args = None): def code(self, code, args=None):
"""Execute arbitrary Python code. 'code' is a formatted string. """Execute arbitrary Python code. 'code' is a formatted string.
It will be run as if it were written into a Python file and It will be run as if it were written into a Python file and
executed. 'args' is a list of strings, and they are passed executed. 'args' is a list of strings, and they are passed
@ -156,23 +152,21 @@ class AppRun(object):
"args must be a list of strings") "args must be a list of strings")
return self.manager.run_code(code, args) return self.manager.run_code(code, args)
class Server(object): class Server(object):
def __init__(self, host = '127.0.0.1', port = 8080, def __init__(self, host='127.0.0.1', port=8080,
embedded = True, # hide diagnostics and output, etc force_traceback=False, # include traceback in all errors
force_traceback = False, # include traceback in all errors basepath='', # base URL path for cherrypy.tree
basepath = '', # base URL path for cherrypy.tree
): ):
self.embedded = embedded
# Build up global server configuration # Build up global server configuration
cherrypy.config.update({ cherrypy.config.update({
'environment': 'embedded',
'server.socket_host': host, 'server.socket_host': host,
'server.socket_port': port, 'server.socket_port': port,
'engine.autoreload_on': False, 'engine.autoreload_on': False,
'server.max_request_body_size': 8*1024*1024, 'server.max_request_body_size': 8*1024*1024,
}) })
if self.embedded:
cherrypy.config.update({ 'environment': 'embedded' })
# Build up application specific configuration # Build up application specific configuration
app_config = {} app_config = {}
@ -181,23 +175,25 @@ class Server(object):
}) })
# Some default headers to just help identify that things are working # Some default headers to just help identify that things are working
app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' }) app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
# Set up Cross-Origin Resource Sharing (CORS) handler so we # Set up Cross-Origin Resource Sharing (CORS) handler so we
# can correctly respond to browsers' CORS preflight requests. # can correctly respond to browsers' CORS preflight requests.
# This also limits verbs to GET and HEAD by default. # This also limits verbs to GET and HEAD by default.
app_config.update({ 'tools.CORS_allow.on': True, app_config.update({
'tools.CORS_allow.methods': ['GET', 'HEAD'] }) 'tools.CORS_allow.on': True,
'tools.CORS_allow.methods': ['GET', 'HEAD']
})
# Configure the 'json_in' tool to also allow other content-types # Configure the 'json_in' tool to also allow other content-types
# (like x-www-form-urlencoded), and to treat JSON as a dict that # (like x-www-form-urlencoded), and to treat JSON as a dict that
# fills requests.param. # fills requests.param.
app_config.update({ 'tools.json_in.force': False, app_config.update({'tools.json_in.force': False,
'tools.json_in.processor': json_to_request_params }) 'tools.json_in.processor': json_to_request_params})
# Send tracebacks in error responses. They're hidden by the # Send tracebacks in error responses. They're hidden by the
# error_page function for client errors (code 400-499). # error_page function for client errors (code 400-499).
app_config.update({ 'request.show_tracebacks' : True }) app_config.update({'request.show_tracebacks': True})
self.force_traceback = force_traceback self.force_traceback = force_traceback
# Patch CherryPy error handler to never pad out error messages. # Patch CherryPy error handler to never pad out error messages.
@ -210,11 +206,12 @@ class Server(object):
manager = serializer_proxy(nilmrun.processmanager.ProcessManager)() manager = serializer_proxy(nilmrun.processmanager.ProcessManager)()
# Build up the application and mount it # Build up the application and mount it
self._manager = manager
root = App() root = App()
root.process = AppProcess(manager) root.process = AppProcess(manager)
root.run = AppRun(manager) root.run = AppRun(manager)
cherrypy.tree.apps = {} cherrypy.tree.apps = {}
cherrypy.tree.mount(root, basepath, config = { "/" : app_config }) cherrypy.tree.mount(root, basepath, config={"/": app_config})
# Set up the WSGI application pointer for external programs # Set up the WSGI application pointer for external programs
self.wsgi_application = cherrypy.tree self.wsgi_application = cherrypy.tree
@ -224,17 +221,20 @@ class Server(object):
return json_error_page(status, message, traceback, version, return json_error_page(status, message, traceback, version,
self.force_traceback) self.force_traceback)
def start(self, blocking = False, event = None): def start(self, blocking=False, event=None):
cherrypy_start(blocking, event, self.embedded) cherrypy_start(blocking, event)
def stop(self): def stop(self):
cherrypy_stop() cherrypy_stop()
# Multiple processes and threads should be OK here, but we'll still # Multiple processes and threads should be OK here, but we'll still
# follow the NilmDB approach of having just one globally initialized # follow the NilmDB approach of having just one globally initialized
# copy of the server object. # copy of the server object.
_wsgi_server = None _wsgi_server = None
def wsgi_application(basepath): # pragma: no cover
def wsgi_application(basepath): # pragma: no cover
"""Return a WSGI application object. """Return a WSGI application object.
'basepath' is the URL path of the application base, which 'basepath' is the URL path of the application base, which
@ -247,13 +247,11 @@ def wsgi_application(basepath): # pragma: no cover
# Try to start the server # Try to start the server
try: try:
_wsgi_server = nilmrun.server.Server( _wsgi_server = nilmrun.server.Server(
embedded = True, basepath=basepath.rstrip('/'))
basepath = basepath.rstrip('/'))
except Exception: except Exception:
# Build an error message on failure # Build an error message on failure
import pprint import pprint
err = sprintf("Initializing nilmrun failed:\n\n", err = "Initializing nilmrun failed:\n\n"
dbpath)
err += traceback.format_exc() err += traceback.format_exc()
try: try:
import pwd import pwd
@ -268,8 +266,10 @@ def wsgi_application(basepath): # pragma: no cover
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ)) err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
if _wsgi_server is None: if _wsgi_server is None:
# Serve up the error with our own mini WSGI app. # Serve up the error with our own mini WSGI app.
headers = [ ('Content-type', 'text/plain'), headers = [
('Content-length', str(len(err))) ] ('Content-type', 'text/plain'),
('Content-length', str(len(err)))
]
start_response("500 Internal Server Error", headers) start_response("500 Internal Server Error", headers)
return [err] return [err]

View File

@ -1,22 +0,0 @@
#!/usr/bin/python
from nilmdb.utils.printf import *
import time
import signal
import sys
# This is just for testing the process management.
def test(n):
n = int(n)
if n < 0: # raise an exception
raise Exception("test exception")
if n == 0: # ignore SIGTERM and count to 100
n = 100
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for x in range(n):
s = sprintf("dummy %d\n", x)
if x & 1:
sys.stdout.write(s)
else:
sys.stderr.write(s)
time.sleep(0.1)

2
requirements.txt Normal file
View File

@ -0,0 +1,2 @@
nilmdb>=2.0.3
psutil==5.7.2

51
scripts/kill.py Executable file
View File

@ -0,0 +1,51 @@
#!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
import nilmrun
import argparse
import os
import sys
def main():
"""Kill/remove a process from the NilmRun server"""
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'Kill/remove a process from the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)
group.add_argument('-n', '--noverify', action="store_true",
help = 'Disable SSL certificate verification')
group = parser.add_argument_group("Program")
group.add_argument('-q', '--quiet', action="store_true",
help = "Don't print out the final log contents")
group.add_argument('pid', nargs='+', help="PIDs to kill")
args = parser.parse_args()
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
# Kill or remove process
all_failed = True
for pid in args.pid:
try:
s = client.post("process/remove", { "pid": pid })
if not args.quiet:
sys.stdout.write(s['log'])
all_failed = False
except ClientError as e:
if "404" in e.status:
fprintf(sys.stderr, "no such pid: %s\n", pid)
else:
raise
# Return error if we failed to remove any of them
if all_failed:
raise SystemExit(1)
if __name__ == "__main__":
main()

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
import nilmrun.server import nilmrun.server
import argparse import argparse
@ -11,9 +11,8 @@ def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Run the NilmRun server', description = 'Run the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter) formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
parser.add_argument("-V", "--version", action="version", version=nilmrun.__version__)
version = nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-a', '--address', group.add_argument('-a', '--address',
@ -36,25 +35,24 @@ def main():
embedded = False embedded = False
server = nilmrun.server.Server(host = args.address, server = nilmrun.server.Server(host = args.address,
port = args.port, port = args.port,
embedded = embedded,
force_traceback = args.traceback) force_traceback = args.traceback)
# Print info # Print info
if not args.quiet: if not args.quiet:
print "NilmRun version: %s" % nilmrun.__version__ print("NilmRun version: %s" % nilmrun.__version__)
print ("Note: This server does not do any authentication! " + print(("Note: This server does not do any authentication! " +
"Anyone who can connect can run arbitrary commands.") "Anyone who can connect can run arbitrary commands."))
if args.address == '0.0.0.0' or args.address == '::': if args.address == '0.0.0.0' or args.address == '::':
host = socket.getfqdn() host = socket.getfqdn()
else: else:
host = args.address host = args.address
print "Server URL: http://%s:%d/" % ( host, args.port) print("Server URL: http://%s:%d/" % ( host, args.port))
print "----" print("----")
server.start(blocking = True) server.start(blocking = True)
if not args.quiet: if not args.quiet:
print "Shutting down" print("Shutting down")
if __name__ == "__main__": if __name__ == "__main__":
main() main()

67
scripts/ps.py Executable file
View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
import datetime_tz
import nilmrun
import argparse
import os
def main():
"""List NilmRun processes"""
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'List NilmRun processes',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)
group.add_argument('-n', '--noverify', action="store_true",
help = 'Disable SSL certificate verification')
args = parser.parse_args()
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
# Print overall system info
info = client.get("process/info")
total = info['total']
system = info['system']
printf(" procs: %d nilm, %d other\n", info['total']['procs'],
info['system']['procs'] - info['total']['procs'])
printf(" cpu: %d%% nilm, %d%% other, %d%% max\n",
round(info['total']['cpu_percent']),
round(info['system']['cpu_percent'] - info['total']['cpu_percent']),
round(info['system']['cpu_max']))
printf(" mem: %d MiB used, %d MiB total, %d%%\n",
round(info['system']['mem_used'] / 1048576.0),
round(info['system']['mem_total'] / 1048576.0),
round(info['system']['mem_used'] * 100.0
/ info['system']['mem_total']))
# Print process detail for each managed process
fmt = "%-36s %-6s %-15s %-4s %-3s %-5s\n"
printf(fmt, "PID", "STATE", "SINCE", "PROC", "CPU", "LOG")
if len(info['pids']) == 0:
printf("No running processes\n")
raise SystemExit(0)
for pid in sorted(info['pids'].keys()):
pidinfo = client.get("process/status", { "pid": pid })
if pidinfo['alive']:
status = "alive"
else:
if pidinfo['exitcode']:
status = "error"
else:
status = "done"
dt = datetime_tz.datetime_tz.fromtimestamp(pidinfo['start_time'])
since = dt.strftime("%m/%d-%H:%M:%S")
printf(fmt, pid, status, since, info['pids'][pid]['procs'],
str(int(round(info['pids'][pid]['cpu_percent']))),
len(pidinfo['log']))
if __name__ == "__main__":
main()

61
scripts/run.py Executable file
View File

@ -0,0 +1,61 @@
#!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
import nilmrun
import argparse
import os
import time
import sys
def main():
"""Run a command on the NilmRun server"""
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'Run a command on the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)
group.add_argument('-n', '--noverify', action="store_true",
help = 'Disable SSL certificate verification')
group = parser.add_argument_group("Program")
group.add_argument('-d', '--detach', action="store_true",
help = 'Run process and return immediately without '
'printing its output')
group.add_argument('cmd', help="Command to run")
group.add_argument('arg', nargs=argparse.REMAINDER,
help="Arguments for command")
args = parser.parse_args()
client = HTTPClient(baseurl=args.url, verify_ssl=not args.noverify,
post_json=True)
# Run command
pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })
# If we're detaching, just print the PID
if args.detach:
print(pid)
raise SystemExit(0)
# Otherwise, watch the log output, and kill the process when it's done
# or when this script terminates.
try:
while True:
s = client.get("process/status", { "pid": pid, "clear": 1 })
sys.stdout.write(s['log'])
sys.stdout.flush()
if not s['alive']:
break
time.sleep(1)
finally:
s = client.post("process/remove", { "pid": pid })
raise SystemExit(s['exitcode'])
if __name__ == "__main__":
main()

View File

@ -20,3 +20,19 @@ cover-erase=1
stop=1 stop=1
verbosity=2 verbosity=2
tests=tests tests=tests
[versioneer]
VCS=git
style=pep440
versionfile_source=nilmrun/_version.py
versionfile_build=nilmrun/_version.py
tag_prefix=nilmrun-
parentdir_prefix=nilmrun-
[flake8]
exclude=_version.py
extend-ignore=E731
[pylint]
ignore=_version.py
disable=C0103,C0111,R0913,R0914

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
# To release a new version, tag it: # To release a new version, tag it:
# git tag -a nilmrun-1.1 -m "Version 1.1" # git tag -a nilmrun-1.1 -m "Version 1.1"
@ -6,66 +6,31 @@
# Then just package it up: # Then just package it up:
# python setup.py sdist # python setup.py sdist
# This is supposed to be using Distribute:
#
# distutils provides a "setup" method.
# setuptools is a set of monkeypatches on top of that.
# distribute is a particular version/implementation of setuptools.
#
# So we don't really know if this is using the old setuptools or the
# Distribute-provided version of setuptools.
import traceback
import sys import sys
import os import os
from setuptools import setup
try:
from setuptools import setup, find_packages
import distutils.version
except ImportError:
traceback.print_exc()
print "Please install the prerequisites listed in README.txt"
sys.exit(1)
# Versioneer manages version numbers from git tags. # Versioneer manages version numbers from git tags.
# https://github.com/warner/python-versioneer # https://github.com/warner/python-versioneer
import versioneer import versioneer
versioneer.versionfile_source = 'nilmrun/_version.py'
versioneer.versionfile_build = 'nilmrun/_version.py'
versioneer.tag_prefix = 'nilmrun-'
versioneer.parentdir_prefix = 'nilmrun-'
# Hack to workaround logging/multiprocessing issue: # Get list of requirements to use in `install_requires` below. Note
# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ # that we don't make a distinction between things that are actually
try: import multiprocessing # required for end-users vs developers (or use `test_requires` or
except: pass # anything else) -- just install everything for simplicity.
install_requires = open('requirements.txt').readlines()
# We need a MANIFEST.in. Generate it here rather than polluting the
# repository with yet another setup-related file.
with open("MANIFEST.in", "w") as m:
m.write("""
# Root
include README.txt
include setup.py
include versioneer.py
include Makefile
""")
# Run setup # Run setup
setup(name='nilmrun', setup(name='nilmrun',
version = versioneer.get_version(), version = versioneer.get_version(),
cmdclass = versioneer.get_cmdclass(), cmdclass = versioneer.get_cmdclass(),
url = 'https://git.jim.sh/jim/lees/nilmrun.git', url = 'https://git.jim.sh/nilm/nilmrun.git',
author = 'Jim Paris', author = 'Jim Paris',
description = "NILM Database Filter Runner", description = "NILM Database Filter Runner",
long_description = "NILM Database Filter Runner", long_description = "NILM Database Filter Runner",
license = "Proprietary", license = "Proprietary",
author_email = 'jim@jtan.com', author_email = 'jim@jtan.com',
install_requires = [ 'nilmdb >= 1.8.3', install_requires = install_requires,
'psutil >= 0.3.0',
'cherrypy >= 3.2',
'simplejson',
],
packages = [ 'nilmrun', packages = [ 'nilmrun',
'nilmrun.scripts', 'nilmrun.scripts',
], ],
@ -75,6 +40,9 @@ setup(name='nilmrun',
entry_points = { entry_points = {
'console_scripts': [ 'console_scripts': [
'nilmrun-server = nilmrun.scripts.nilmrun_server:main', 'nilmrun-server = nilmrun.scripts.nilmrun_server:main',
'nilmrun-ps = nilmrun.scripts.ps:main',
'nilmrun-run = nilmrun.scripts.run:main',
'nilmrun-kill = nilmrun.scripts.kill:main',
], ],
}, },
zip_safe = False, zip_safe = False,

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
import nose import nose
import os import os

View File

@ -15,14 +15,15 @@ import distutils.version
import os import os
import sys import sys
import threading import threading
import cStringIO import io
import simplejson as json import json
import unittest import unittest
import warnings import warnings
import time import time
import re import re
import urllib2 import urllib.request, urllib.error, urllib.parse
from urllib2 import urlopen, HTTPError from urllib.request import urlopen
from urllib.error import HTTPError
import requests import requests
import pprint import pprint
import textwrap import textwrap
@ -128,10 +129,27 @@ class TestClient(object):
def _run_testfilter(self, client, args): def _run_testfilter(self, client, args):
code = textwrap.dedent(""" code = textwrap.dedent("""
import nilmrun.testfilter from nilmdb.utils.printf import *
import simplejson as json import time
import signal
import json
import sys import sys
nilmrun.testfilter.test(json.loads(sys.argv[1])) # This is just for testing the process management.
def test(n):
n = int(n)
if n < 0: # raise an exception
raise Exception("test exception")
if n == 0: # ignore SIGTERM and count to 100
n = 100
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for x in range(n):
s = sprintf("dummy %d\\n", x)
if x & 1:
sys.stdout.write(s)
else:
sys.stderr.write(s)
time.sleep(0.1)
test(json.loads(sys.argv[1]))
""") """)
jsonargs = json.dumps(args) jsonargs = json.dumps(args)
return client.post("run/code", { "code": code, "args": [ jsonargs ] }) return client.post("run/code", { "code": code, "args": [ jsonargs ] })
@ -269,9 +287,9 @@ class TestClient(object):
# basic code snippet # basic code snippet
code = textwrap.dedent(""" code = textwrap.dedent("""
print 'hello' print('hello')
def foo(arg): def foo(arg):
print 'world' print('world')
""") """)
status = do(code, [], False) status = do(code, [], False)
eq_("hello\n", status["log"]) eq_("hello\n", status["log"])
@ -280,7 +298,7 @@ class TestClient(object):
# compile error # compile error
code = textwrap.dedent(""" code = textwrap.dedent("""
def foo(arg: def foo(arg:
print 'hello' print('hello')
""") """)
status = do(code, [], False) status = do(code, [], False)
in_("SyntaxError", status["log"]) in_("SyntaxError", status["log"])
@ -305,11 +323,11 @@ class TestClient(object):
# argument handling (strings come in as unicode) # argument handling (strings come in as unicode)
code = textwrap.dedent(""" code = textwrap.dedent("""
import sys import sys
print sys.argv[1].encode('ascii'), sys.argv[2] print(sys.argv[1], sys.argv[2])
sys.exit(0) # also test raising SystemExit sys.exit(0) # also test raising SystemExit
""") """)
with assert_raises(ClientError) as e: with assert_raises(ClientError) as e:
do(code, ["hello", 123], False) do(code, ["hello", 123], False)
in_("400 Bad Request", str(e.exception)) in_("400 Bad Request", str(e.exception))
status = do(code, ["hello", "123"], False) status = do(code, ["hello", "123"], False)
eq_(status["log"], "hello 123\n") eq_(status["log"], "hello 123\n")
@ -318,9 +336,9 @@ class TestClient(object):
# try killing a long-running process # try killing a long-running process
code = textwrap.dedent(""" code = textwrap.dedent("""
import time import time
print 'hello' print('hello')
time.sleep(60) time.sleep(60)
print 'world' print('world')
""") """)
status = do(code, [], True) status = do(code, [], True)
eq_(status["log"], "hello\n") eq_(status["log"], "hello\n")
@ -329,7 +347,7 @@ class TestClient(object):
# default arguments are empty # default arguments are empty
code = textwrap.dedent(""" code = textwrap.dedent("""
import sys import sys
print 'args:', len(sys.argv[1:]) print('args:', len(sys.argv[1:]))
""") """)
status = do(code, None, False) status = do(code, None, False)
eq_(status["log"], "args: 0\n") eq_(status["log"], "args: 0\n")
@ -352,8 +370,10 @@ class TestClient(object):
# start some processes # start some processes
a = client.post("run/command", { "argv": ["sleep","60"] } ) a = client.post("run/command", { "argv": ["sleep","60"] } )
b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } ) b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } )
c = client.post("run/command", { "argv": ["sh","-c","burnP5;true"] } ) c = client.post("run/command", { "argv": [
d = client.post("run/command", { "argv": ["burnP5" ] } ) "sh","-c","dd if=/dev/zero of=/dev/null;true"] } )
d = client.post("run/command", { "argv": [
"dd", "if=/dev/zero", "of=/dev/null" ] } )
info = client.get("process/info") info = client.get("process/info")
eq_(info["pids"][a]["procs"], 1) eq_(info["pids"][a]["procs"], 1)
@ -365,9 +385,13 @@ class TestClient(object):
lt_(20, info["pids"][c]["cpu_percent"]) lt_(20, info["pids"][c]["cpu_percent"])
lt_(80, info["system"]["cpu_percent"]) lt_(80, info["system"]["cpu_percent"])
time.sleep(2) for x in range(10):
info = client.get("process/info") time.sleep(1)
eq_(info["pids"][b]["procs"], 0) info = client.get("process/info")
if info["pids"][b]["procs"] != 2:
break
else:
raise Exception("process B didn't die: " + str(info["pids"][b]))
# kill all processes # kill all processes
for pid in client.get("process/list"): for pid in client.get("process/list"):
@ -377,16 +401,34 @@ class TestClient(object):
client = HTTPClient(baseurl = testurl, post_json = True) client = HTTPClient(baseurl = testurl, post_json = True)
eq_(client.get("process/list"), []) eq_(client.get("process/list"), [])
def verify(cmd, result): def verify(cmd, result):
pid = client.post("run/command", { "argv": [ "sh", "-c", cmd ] }) pid = client.post("run/command", { "argv": [
"/bin/bash", "-c", cmd ] })
eq_(client.get("process/list"), [pid]) eq_(client.get("process/list"), [pid])
status = self.wait_end(client, pid) status = self.wait_end(client, pid)
eq_(result, status["log"]) eq_(result, status["log"])
# Unicode should work # Unicode should work
verify("echo -n hello", "hello") verify("echo -n hello", "hello")
verify(u"echo -n ☠", u"") verify("echo -n ☠", "")
verify("echo -ne \\\\xe2\\\\x98\\\\xa0", u"") verify("echo -ne \\\\xe2\\\\x98\\\\xa0", "")
# Programs that spit out invalid UTF-8 should get replacement # Programs that spit out invalid UTF-8 should get replacement
# markers # markers
verify("echo -ne \\\\xae", u"\ufffd") verify("echo -ne \\\\xae", "\ufffd")
def test_client_11_atexit(self):
# Leave a directory and running process behind, for the atexit
# handler to clean up. Here we trigger the atexit manually,
# since it's hard to trigger it as part of the test suite.
client = HTTPClient(baseurl = testurl, post_json = True)
code = textwrap.dedent("""
import time
time.sleep(10)
""")
client.post("run/code", { "code": code, "args": [ "hello"] })
# Trigger atexit function
test_server._manager._atexit()
# Ensure no processes exit
eq_(client.get("process/list"), [])

View File

@ -1,7 +1,7 @@
# Just some helpers for test functions # Just some helpers for test functions
def myrepr(x): def myrepr(x):
if isinstance(x, basestring): if isinstance(x, str):
return '"' + x + '"' return '"' + x + '"'
else: else:
return repr(x) return repr(x)

File diff suppressed because it is too large Load Diff