nilm/nilmrun
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: nilm:master
Branches Tags
nilm:master
nilm:nilmrun-2.0.3
nilm:nilmrun-2.0.2
nilm:nilmrun-2.0.1
nilm:nilmrun-2.0.0
nilm:nilmrun-1.3.3
nilm:nilmrun-1.3.2
nilm:nilmrun-1.3.1
nilm:nilmrun-1.3.0
nilm:nilmrun-1.2.5
nilm:nilmrun-1.2.4
nilm:nilmrun-1.2.3
nilm:nilmrun-1.2.2
nilm:nilmrun-1.2.1
nilm:nilmrun-1.2.0
nilm:nilmrun-1.1.2
nilm:nilmrun-1.1.1
nilm:nilmrun-1.1.0
nilm:nilmrun-1.0.2
nilm:nilmrun-1.0.1
nilm:nilmrun-1.0.0
nilm:nilmrun-0.3
nilm:nilmrun-0.2
nilm:nilmrun-0.1
..
compare: nilm:nilmrun-1.2.4
Branches Tags
nilm:master
nilm:nilmrun-2.0.3
nilm:nilmrun-2.0.2
nilm:nilmrun-2.0.1
nilm:nilmrun-2.0.0
nilm:nilmrun-1.3.3
nilm:nilmrun-1.3.2
nilm:nilmrun-1.3.1
nilm:nilmrun-1.3.0
nilm:nilmrun-1.2.5
nilm:nilmrun-1.2.4
nilm:nilmrun-1.2.3
nilm:nilmrun-1.2.2
nilm:nilmrun-1.2.1
nilm:nilmrun-1.2.0
nilm:nilmrun-1.1.2
nilm:nilmrun-1.1.1
nilm:nilmrun-1.1.0
nilm:nilmrun-1.0.2
nilm:nilmrun-1.0.1
nilm:nilmrun-1.0.0
nilm:nilmrun-0.3
nilm:nilmrun-0.2
nilm:nilmrun-0.1

No commits in common. "master" and "nilmrun-1.2.4" have entirely different histories.
master ... nilmrun-1.

22 changed files with 896 additions and 2626 deletions
Show Stats Expand all files Collapse all files

2
.gitattributes vendored
View File

@ -1 +1 @@
nilmrun/_version.py export-subst
src/_version.py export-subst

2
.gitignore vendored
View File

@ -3,8 +3,8 @@ build/
*.pyc
dist/
nilmrun.egg-info/
.eggs/
# This gets generated as needed by setup.py
MANIFEST.in
MANIFEST

8
MANIFEST.in
View File

@ -1,8 +0,0 @@
# Root
include README.md
include setup.py
include versioneer.py
include Makefile
# Version
include nilmrun/_version.py

33
Makefile
View File

@ -1,49 +1,48 @@
# By default, run the tests.
all: test
test2:
nilmrun/trainola.py data.js
version:
python3 setup.py version
python setup.py version
build:
python3 setup.py build_ext --inplace
python setup.py build_ext --inplace
dist: sdist
sdist:
python3 setup.py sdist
python setup.py sdist
install:
python3 setup.py install
python setup.py install
develop:
python3 setup.py develop
python setup.py develop
docs:
make -C docs
ctrl: flake
flake:
flake8 nilmrun
lint:
pylint3 --rcfile=setup.cfg nilmrun
pylint --rcfile=.pylintrc nilmdb
test:
ifneq ($(INSIDE_EMACS),)
ifeq ($(INSIDE_EMACS), t)
# Use the slightly more flexible script
python3 setup.py build_ext --inplace
python3 tests/runtests.py
python setup.py build_ext --inplace
python tests/runtests.py
else
# Let setup.py check dependencies, build stuff, and run the test
python3 setup.py nosetests
python setup.py nosetests
endif
clean::
find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
rm -f .coverage
rm -rf nilmrun.egg-info/ build/
find . -name '*pyc' | xargs rm -f
rm -rf nilmtools.egg-info/ build/ MANIFEST.in
make -C docs clean
gitclean::
git clean -dXf
.PHONY: all version dist sdist install docs test
.PHONY: ctrl lint flake clean gitclean
.PHONY: all version dist sdist install docs lint test clean gitclean

32
README.md
View File

@ -1,32 +0,0 @@
# nilmrun: Run NilmDB filters
by Jim Paris <jim@jtan.com>
## Prerequisites:
# Runtime and build environments
sudo apt-get install python3
# Create a new Python virtual environment to isolate deps.
python3 -m venv ../venv
source ../venv/bin/activate # run "deactivate" to leave
# Install all Python dependencies
pip3 install -r requirements.txt
## Install:
Install it into the virtual environment
python3 setup.py install
If you want to instead install it system-wide, you will also need to
install the requirements system-wide:
sudo pip3 install -r requirements.txt
sudo python3 setup.py install
## Usage:
nilmrun-server --help
See docs/wsgi.md for info on setting up a WSGI application in Apache.

20
README.txt Normal file
View File

@ -0,0 +1,20 @@
nilmrun: Run NilmDB filters
by Jim Paris <jim@jtan.com>
Prerequisites:
# Runtime and build environments
sudo apt-get install python2.7 python-setuptools
# Plus nilmdb and its dependencies
nilmdb (1.8.3+)
Install:
python setup.py install
Usage:
nilmrun-server --help
See docs/wsgi.md for info on setting up a WSGI application in Apache.

2
nilmrun/__init__.py
View File

@ -1,3 +1,5 @@
import nilmrun.processmanager
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

607
nilmrun/_version.py
View File

@ -1,520 +1,197 @@
IN_LONG_VERSION_PY = True
# This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag
# feature). Distribution tarballs (built by setup.py sdist) and build
# feature). Distribution tarballs (build by setup.py sdist) and build
# directories (produced by setup.py build) will contain a much shorter file
# that just contains the computed version number.
# This file is released into the public domain. Generated by
# versioneer-0.18 (https://github.com/warner/python-versioneer)
# versioneer-0.7+ (https://github.com/warner/python-versioneer)
# these strings will be replaced by git during git-archive
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
"""Git implementation of _version.py."""
import errno
import os
import re
import subprocess
import sys
def get_keywords():
"""Get the keywords needed to look up the version information."""
# these strings will be replaced by git during git-archive.
# setup.py/versioneer.py will grep for the variable names, so they must
# each be defined on a line of their own. _version.py will just call
# get_keywords().
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
git_date = "$Format:%ci$"
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
return keywords
class VersioneerConfig:
"""Container for Versioneer configuration parameters."""
def get_config():
"""Create, populate and return the VersioneerConfig() object."""
# these strings are filled in when 'setup.py versioneer' creates
# _version.py
cfg = VersioneerConfig()
cfg.VCS = "git"
cfg.style = "pep440"
cfg.tag_prefix = "nilmrun-"
cfg.parentdir_prefix = "nilmrun-"
cfg.versionfile_source = "nilmrun/_version.py"
cfg.verbose = False
return cfg
class NotThisMethod(Exception):
"""Exception raised if a method is not valid for the current scenario."""
LONG_VERSION_PY = {}
HANDLERS = {}
def register_vcs_handler(vcs, method): # decorator
"""Decorator to mark a method as the handler for a particular VCS."""
def decorate(f):
"""Store f in HANDLERS[vcs][method]."""
if vcs not in HANDLERS:
HANDLERS[vcs] = {}
HANDLERS[vcs][method] = f
return f
return decorate
def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
env=None):
"""Call the given command(s)."""
assert isinstance(commands, list)
p = None
for c in commands:
def run_command(args, cwd=None, verbose=False):
try:
dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git
p = subprocess.Popen([c] + args, cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
break
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
except EnvironmentError:
e = sys.exc_info()[1]
if e.errno == errno.ENOENT:
continue
if verbose:
print("unable to run %s" % dispcmd)
print("unable to run %s" % args[0])
print(e)
return None, None
else:
if verbose:
print("unable to find command, tried %s" % (commands,))
return None, None
return None
stdout = p.communicate()[0].strip()
if sys.version_info[0] >= 3:
if sys.version >= '3':
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
print("unable to run %s (error)" % dispcmd)
print("stdout was %s" % stdout)
return None, p.returncode
return stdout, p.returncode
print("unable to run %s (error)" % args[0])
return None
return stdout
def versions_from_parentdir(parentdir_prefix, root, verbose):
"""Try to determine the version from the parent directory name.
import sys
import re
import os.path
Source tarballs conventionally unpack into a directory that includes both
the project name and a version string. We will also support searching up
two directory levels for an appropriately named parent directory
"""
rootdirs = []
for i in range(3):
dirname = os.path.basename(root)
if dirname.startswith(parentdir_prefix):
return {"version": dirname[len(parentdir_prefix):],
"full-revisionid": None,
"dirty": False, "error": None, "date": None}
else:
rootdirs.append(root)
root = os.path.dirname(root) # up a level
if verbose:
print("Tried directories %s but none started with prefix %s" %
(str(rootdirs), parentdir_prefix))
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
@register_vcs_handler("git", "get_keywords")
def git_get_keywords(versionfile_abs):
"""Extract version information from the given file."""
def get_expanded_variables(versionfile_source):
# the code embedded in _version.py can just fetch the value of these
# keywords. When used from setup.py, we don't want to import _version.py,
# so we do it with a regexp instead. This function is not used from
# _version.py.
keywords = {}
# variables. When used from setup.py, we don't want to import
# _version.py, so we do it with a regexp instead. This function is not
# used from _version.py.
variables = {}
try:
f = open(versionfile_abs, "r")
for line in f.readlines():
for line in open(versionfile_source,"r").readlines():
if line.strip().startswith("git_refnames ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["refnames"] = mo.group(1)
variables["refnames"] = mo.group(1)
if line.strip().startswith("git_full ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["full"] = mo.group(1)
if line.strip().startswith("git_date ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["date"] = mo.group(1)
f.close()
variables["full"] = mo.group(1)
except EnvironmentError:
pass
return keywords
return variables
@register_vcs_handler("git", "keywords")
def git_versions_from_keywords(keywords, tag_prefix, verbose):
"""Get version information from git keywords."""
if not keywords:
raise NotThisMethod("no keywords at all, weird")
date = keywords.get("date")
if date is not None:
# git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
# datestamp. However we prefer "%ci" (which expands to an "ISO-8601
# -like" string, which we must then edit to make compliant), because
# it's been around since git-1.5.3, and it's too difficult to
# discover which version we're using, or to work around using an
# older one.
date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
refnames = keywords["refnames"].strip()
def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
refnames = variables["refnames"].strip()
if refnames.startswith("$Format"):
if verbose:
print("keywords are unexpanded, not using")
raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
print("variables are unexpanded, not using")
return {} # unexpanded, so not in an unpacked git-archive tarball
refs = set([r.strip() for r in refnames.strip("()").split(",")])
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
TAG = "tag: "
tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
if not tags:
# Either we're using git < 1.8.3, or there really are no tags. We use
# a heuristic: assume all version tags have a digit. The old git %d
# expansion behaves like git log --decorate=short and strips out the
# refs/heads/ and refs/tags/ prefixes that would let us distinguish
# between branches and tags. By ignoring refnames without digits, we
# filter out many common branch names like "release" and
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
for ref in list(refs):
if not re.search(r'\d', ref):
if verbose:
print("discarding '%s', no digits" % ",".join(refs - tags))
print("discarding '%s', no digits" % ref)
refs.discard(ref)
# Assume all version tags have a digit. git's %d expansion
# behaves like git log --decorate=short and strips out the
# refs/heads/ and refs/tags/ prefixes that would let us
# distinguish between branches and tags. By ignoring refnames
# without digits, we filter out many common branch names like
# "release" and "stabilization", as well as "HEAD" and "master".
if verbose:
print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(tags):
print("remaining refs: %s" % ",".join(sorted(refs)))
for ref in sorted(refs):
# sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):]
if verbose:
print("picking %s" % r)
return {"version": r,
"full-revisionid": keywords["full"].strip(),
"dirty": False, "error": None,
"date": date}
# no suitable tags, so version is "0+unknown", but full hex is still there
return { "version": r,
"full": variables["full"].strip() }
# no suitable tags, so we use the full revision id
if verbose:
print("no suitable tags, using unknown + full revision id")
return {"version": "0+unknown",
"full-revisionid": keywords["full"].strip(),
"dirty": False, "error": "no suitable tags", "date": None}
print("no suitable tags, using full revision id")
return { "version": variables["full"].strip(),
"full": variables["full"].strip() }
@register_vcs_handler("git", "pieces_from_vcs")
def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
"""Get version from 'git describe' in the root of the source tree.
This only gets called if the git-archive 'subst' keywords were *not*
expanded, and _version.py hasn't already been rewritten with a short
version string, meaning we're inside a checked out source tree.
"""
GITS = ["git"]
if sys.platform == "win32":
GITS = ["git.cmd", "git.exe"]
out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
hide_stderr=True)
if rc != 0:
if verbose:
print("Directory %s not under git control" % root)
raise NotThisMethod("'git rev-parse --git-dir' returned error")
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
# if there isn't one, this yields HEX[-dirty] (no NUM)
describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
"--always", "--long",
"--match", "%s*" % tag_prefix],
cwd=root)
# --long was added in git-1.5.5
if describe_out is None:
raise NotThisMethod("'git describe' failed")
describe_out = describe_out.strip()
full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
if full_out is None:
raise NotThisMethod("'git rev-parse' failed")
full_out = full_out.strip()
pieces = {}
pieces["long"] = full_out
pieces["short"] = full_out[:7] # maybe improved later
pieces["error"] = None
# parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
# TAG might have hyphens.
git_describe = describe_out
# look for -dirty suffix
dirty = git_describe.endswith("-dirty")
pieces["dirty"] = dirty
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
# now we have TAG-NUM-gHEX or HEX
if "-" in git_describe:
# TAG-NUM-gHEX
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
if not mo:
# unparseable. Maybe git-describe is misbehaving?
pieces["error"] = ("unable to parse git-describe output: '%s'"
% describe_out)
return pieces
# tag
full_tag = mo.group(1)
if not full_tag.startswith(tag_prefix):
if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'"
print(fmt % (full_tag, tag_prefix))
pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
% (full_tag, tag_prefix))
return pieces
pieces["closest-tag"] = full_tag[len(tag_prefix):]
# distance: number of commits since tag
pieces["distance"] = int(mo.group(2))
# commit: short hex revision ID
pieces["short"] = mo.group(3)
else:
# HEX: no tags
pieces["closest-tag"] = None
count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
cwd=root)
pieces["distance"] = int(count_out) # total number of commits
# commit date: see ISO-8601 comment in git_versions_from_keywords()
date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
cwd=root)[0].strip()
pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
return pieces
def plus_or_dot(pieces):
"""Return a + if we don't already have one, else return a ."""
if "+" in pieces.get("closest-tag", ""):
return "."
return "+"
def render_pep440(pieces):
"""Build up version string, with post-release "local version identifier".
Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
Exceptions:
1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += plus_or_dot(pieces)
rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
else:
# exception #1
rendered = "0+untagged.%d.g%s" % (pieces["distance"],
pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
return rendered
def render_pep440_pre(pieces):
"""TAG[.post.devDISTANCE] -- No -dirty.
Exceptions:
1: no tags. 0.post.devDISTANCE
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"]:
rendered += ".post.dev%d" % pieces["distance"]
else:
# exception #1
rendered = "0.post.dev%d" % pieces["distance"]
return rendered
def render_pep440_post(pieces):
"""TAG[.postDISTANCE[.dev0]+gHEX] .
The ".dev0" means dirty. Note that .dev0 sorts backwards
(a dirty tree will appear "older" than the corresponding clean one),
but you shouldn't be releasing software with -dirty anyways.
Exceptions:
1: no tags. 0.postDISTANCE[.dev0]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += ".post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
rendered += plus_or_dot(pieces)
rendered += "g%s" % pieces["short"]
else:
# exception #1
rendered = "0.post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
rendered += "+g%s" % pieces["short"]
return rendered
def render_pep440_old(pieces):
"""TAG[.postDISTANCE[.dev0]] .
The ".dev0" means dirty.
Eexceptions:
1: no tags. 0.postDISTANCE[.dev0]
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"] or pieces["dirty"]:
rendered += ".post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
else:
# exception #1
rendered = "0.post%d" % pieces["distance"]
if pieces["dirty"]:
rendered += ".dev0"
return rendered
def render_git_describe(pieces):
"""TAG[-DISTANCE-gHEX][-dirty].
Like 'git describe --tags --dirty --always'.
Exceptions:
1: no tags. HEX[-dirty] (note: no 'g' prefix)
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
if pieces["distance"]:
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
else:
# exception #1
rendered = pieces["short"]
if pieces["dirty"]:
rendered += "-dirty"
return rendered
def render_git_describe_long(pieces):
"""TAG-DISTANCE-gHEX[-dirty].
Like 'git describe --tags --dirty --always -long'.
The distance/hash is unconditional.
Exceptions:
1: no tags. HEX[-dirty] (note: no 'g' prefix)
"""
if pieces["closest-tag"]:
rendered = pieces["closest-tag"]
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
else:
# exception #1
rendered = pieces["short"]
if pieces["dirty"]:
rendered += "-dirty"
return rendered
def render(pieces, style):
"""Render the given version pieces into the requested style."""
if pieces["error"]:
return {"version": "unknown",
"full-revisionid": pieces.get("long"),
"dirty": None,
"error": pieces["error"],
"date": None}
if not style or style == "default":
style = "pep440" # the default
if style == "pep440":
rendered = render_pep440(pieces)
elif style == "pep440-pre":
rendered = render_pep440_pre(pieces)
elif style == "pep440-post":
rendered = render_pep440_post(pieces)
elif style == "pep440-old":
rendered = render_pep440_old(pieces)
elif style == "git-describe":
rendered = render_git_describe(pieces)
elif style == "git-describe-long":
rendered = render_git_describe_long(pieces)
else:
raise ValueError("unknown style '%s'" % style)
return {"version": rendered, "full-revisionid": pieces["long"],
"dirty": pieces["dirty"], "error": None,
"date": pieces.get("date")}
def get_versions():
"""Get version information or return default if unable to do so."""
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
# __file__, we can work backwards from there to the root. Some
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
# case we can only use expanded keywords.
cfg = get_config()
verbose = cfg.verbose
def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
# this runs 'git' from the root of the source tree. That either means
# someone ran a setup.py command (and this code is in versioneer.py, so
# IN_LONG_VERSION_PY=False, thus the containing directory is the root of
# the source tree), or someone ran a project-specific entry point (and
# this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
# containing directory is somewhere deeper in the source tree). This only
# gets called if the git-archive 'subst' variables were *not* expanded,
# and _version.py hasn't already been rewritten with a short version
# string, meaning we're inside a checked out source tree.
try:
return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
verbose)
except NotThisMethod:
pass
try:
root = os.path.realpath(__file__)
# versionfile_source is the relative path from the top of the source
# tree (where the .git directory might live) to this file. Invert
# this to find the root from __file__.
for i in cfg.versionfile_source.split('/'):
root = os.path.dirname(root)
here = os.path.abspath(__file__)
except NameError:
return {"version": "0+unknown", "full-revisionid": None,
"dirty": None,
"error": "unable to find root of source tree",
"date": None}
# some py2exe/bbfreeze/non-CPython implementations don't do __file__
return {} # not always correct
# versionfile_source is the relative path from the top of the source tree
# (where the .git directory might live) to this file. Invert this to find
# the root from __file__.
root = here
if IN_LONG_VERSION_PY:
for i in range(len(versionfile_source.split("/"))):
root = os.path.dirname(root)
else:
root = os.path.dirname(here)
if not os.path.exists(os.path.join(root, ".git")):
if verbose:
print("no .git in %s" % root)
return {}
GIT = "git"
if sys.platform == "win32":
GIT = "git.cmd"
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
cwd=root)
if stdout is None:
return {}
if not stdout.startswith(tag_prefix):
if verbose:
print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
return {}
tag = stdout[len(tag_prefix):]
stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
if stdout is None:
return {}
full = stdout.strip()
if tag.endswith("-dirty"):
full += "-dirty"
return {"version": tag, "full": full}
def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
if IN_LONG_VERSION_PY:
# We're running from _version.py. If it's from a source tree
# (execute-in-place), we can work upwards to find the root of the
# tree, and then check the parent directory for a version string. If
# it's in an installed application, there's no hope.
try:
pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
return render(pieces, cfg.style)
except NotThisMethod:
pass
here = os.path.abspath(__file__)
except NameError:
# py2exe/bbfreeze/non-CPython don't have __file__
return {} # without __file__, we have no hope
# versionfile_source is the relative path from the top of the source
# tree to _version.py. Invert this to find the root from __file__.
root = here
for i in range(len(versionfile_source.split("/"))):
root = os.path.dirname(root)
else:
# we're running from versioneer.py, which means we're running from
# the setup.py in a source tree. sys.argv[0] is setup.py in the root.
here = os.path.abspath(sys.argv[0])
root = os.path.dirname(here)
try:
if cfg.parentdir_prefix:
return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
except NotThisMethod:
pass
# Source tarballs conventionally unpack into a directory that includes
# both the project name and a version string.
dirname = os.path.basename(root)
if not dirname.startswith(parentdir_prefix):
if verbose:
print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
(root, dirname, parentdir_prefix))
return None
return {"version": dirname[len(parentdir_prefix):], "full": ""}
tag_prefix = "nilmrun-"
parentdir_prefix = "nilmrun-"
versionfile_source = "nilmrun/_version.py"
def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
variables = { "refnames": git_refnames, "full": git_full }
ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
if not ver:
ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
if not ver:
ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
verbose)
if not ver:
ver = default
return ver
return {"version": "0+unknown", "full-revisionid": None,
"dirty": None,
"error": "unable to compute version", "date": None}

140
nilmrun/processmanager.py
View File

@ -1,8 +1,10 @@
#!/usr/bin/env python3
#!/usr/bin/python
from nilmdb.utils.printf import *
import threading
import subprocess
import io
import cStringIO
import sys
import os
import signal
@ -13,18 +15,16 @@ import tempfile
import atexit
import shutil
class ProcessError(Exception):
pass
class LogReceiver(object):
"""Spawn a thread that listens to a pipe for log messages,
and stores them locally."""
def __init__(self, pipe):
self.pipe = pipe
self.log = io.BytesIO()
self.thread = threading.Thread(target=self.run)
self.log = cStringIO.StringIO()
self.thread = threading.Thread(target = self.run)
self.thread.start()
def run(self):
@ -39,12 +39,11 @@ class LogReceiver(object):
return self.log.getvalue()
def clear(self):
self.log = io.BytesIO()
self.log = cStringIO.StringIO()
class Process(object):
"""Spawn and manage a subprocess, and capture its output."""
def __init__(self, argv, tempfile=None):
def __init__(self, argv, tempfile = None):
self.start_time = None
# Use a pipe for communicating log data
@ -56,9 +55,9 @@ class Process(object):
# Spawn the new process
try:
self._process = subprocess.Popen(args=argv, stdin=nullfd,
stdout=wpipe, stderr=wpipe,
close_fds=True, cwd="/tmp")
self._process = subprocess.Popen(args = argv, stdin = nullfd,
stdout = wpipe, stderr = wpipe,
close_fds = True, cwd = "/tmp")
except (OSError, TypeError) as e:
raise ProcessError(str(e))
finally:
@ -70,7 +69,7 @@ class Process(object):
self.start_time = time.time()
self.pid = str(uuid.uuid1(self._process.pid or 0))
def _join(self, timeout=1.0):
def _join(self, timeout = 1.0):
start = time.time()
while True:
if self._process.poll() is not None:
@ -79,7 +78,7 @@ class Process(object):
return False
time.sleep(0.1)
def terminate(self, timeout=1.0):
def terminate(self, timeout = 1.0):
"""Terminate a process, and all of its children that are in the same
process group."""
try:
@ -93,21 +92,15 @@ class Process(object):
except OSError: # pragma: no cover
return None
def kill(pid, sig):
try:
return os.kill(pid, sig)
except OSError: # pragma: no cover
return
# Find all children
group = getpgid(self._process.pid)
main = psutil.Process(self._process.pid)
allproc = [main] + main.children(recursive=True)
allproc = [ main ] + main.get_children(recursive = True)
# Kill with SIGTERM, if they're still in this process group
for proc in allproc:
if getpgid(proc.pid) == group:
kill(proc.pid, signal.SIGTERM)
os.kill(proc.pid, signal.SIGTERM)
# Wait for it to die again
if self._join(timeout):
@ -116,7 +109,7 @@ class Process(object):
# One more try with SIGKILL
for proc in allproc:
if getpgid(proc.pid) == group:
kill(proc.pid, signal.SIGKILL)
os.kill(proc.pid, signal.SIGKILL)
# See if it worked
return self._join(timeout)
@ -143,22 +136,22 @@ class Process(object):
Call .get_info() about a second later."""
try:
main = psutil.Process(self._process.pid)
self._process_list = [main] + main.children(recursive=True)
self._process_list = [ main ] + main.get_children(recursive = True)
for proc in self._process_list:
proc.cpu_percent(0)
proc.get_cpu_percent(0)
except psutil.Error: # pragma: no cover (race condition)
self._process_list = []
self._process_list = [ ]
@staticmethod
def get_empty_info():
return {"cpu_percent": 0,
return { "cpu_percent": 0,
"cpu_user": 0,
"cpu_sys": 0,
"mem_phys": 0,
"mem_virt": 0,
"io_read": 0,
"io_write": 0,
"procs": 0}
"procs": 0 }
def get_info(self):
"""Return a dictionary with info about the process CPU and memory
@ -166,14 +159,14 @@ class Process(object):
d = self.get_empty_info()
for proc in self._process_list:
try:
d["cpu_percent"] += proc.cpu_percent(0)
cpuinfo = proc.cpu_times()
d["cpu_percent"] += proc.get_cpu_percent(0)
cpuinfo = proc.get_cpu_times()
d["cpu_user"] += cpuinfo.user
d["cpu_sys"] += cpuinfo.system
meminfo = proc.memory_info()
meminfo = proc.get_memory_info()
d["mem_phys"] += meminfo.rss
d["mem_virt"] += meminfo.vms
ioinfo = proc.io_counters()
ioinfo = proc.get_io_counters()
d["io_read"] += ioinfo.read_bytes
d["io_write"] += ioinfo.write_bytes
d["procs"] += 1
@ -181,35 +174,16 @@ class Process(object):
pass
return d
class ProcessManager(object):
"""Track and manage a collection of Process objects"""
def __init__(self):
self.processes = {}
self.tmpdirs = {}
atexit.register(self._atexit)
def _cleanup_tmpdir(self, pid):
if pid in self.tmpdirs:
try:
shutil.rmtree(self.tmpdirs[pid])
except OSError: # pragma: no cover
pass
del self.tmpdirs[pid]
def _atexit(self):
# Kill remaining processes, remove their dirs
for pid in list(self.processes.keys()):
try:
self.processes[pid].terminate()
del self.processes[pid]
shutil.rmtree(self.tmpdirs[pid])
del self.tmpdirs[pid]
except Exception: # pragma: no cover
pass
self.tmpfiles = {}
self.tmpdir = tempfile.mkdtemp(prefix = "nilmrun-usercode-")
atexit.register(shutil.rmtree, self.tmpdir)
def __iter__(self):
return iter(list(self.processes.keys()))
return iter(self.processes.keys())
def __getitem__(self, key):
return self.processes[key]
@ -219,33 +193,15 @@ class ProcessManager(object):
executed. The arguments, which must be strings, will be
accessible in the code as sys.argv[1:]."""
# The easiest way to do this, by far, is to just write the
# code to a file. Make a directory to put it in.
tmpdir = tempfile.mkdtemp(prefix="nilmrun-usercode-")
try:
# Write the code
codepath = os.path.join(tmpdir, "usercode.py")
with open(codepath, "w") as f:
# code to a file.
(fd, path) = tempfile.mkstemp(prefix = "nilmrun-usercode-",
suffix = ".py", dir=self.tmpdir)
with os.fdopen(fd, 'w') as f:
f.write(code)
# Save the args too, for debugging purposes
with open(os.path.join(tmpdir, "args.txt"), "w") as f:
f.write(repr(args))
# Run the code
argv = [sys.executable, "-B", "-s", "-u", codepath] + args
argv = [ sys.executable, "-B", "-s", "-u", path ] + args
pid = self.run_command(argv)
# Save the temp dir
self.tmpdirs[pid] = tmpdir
tmpdir = None # Don't need to remove it anymore
self.tmpfiles[pid] = path
return pid
finally:
# Clean up tempdir if we didn't finish
if tmpdir is not None:
try:
shutil.rmtree(tmpdir)
except OSError: # pragma: no cover
pass
def run_command(self, argv):
"""Execute a command line program"""
@ -257,32 +213,38 @@ class ProcessManager(object):
return self.processes[pid].terminate()
def remove(self, pid):
self._cleanup_tmpdir(pid)
if pid in self.tmpfiles:
try:
os.unlink(self.tmpfiles[pid])
except OSError: # pragma: no cover
pass
del self.tmpfiles[pid]
del self.processes[pid]
def get_info(self):
"""Get info about all running PIDs"""
info = {
"total": Process.get_empty_info(),
"pids": {},
"system": {}
info = { "total" : Process.get_empty_info(),
"pids" : {},
"system" : {}
}
# Trigger CPU usage collection
for pid in self:
self[pid].get_info_prepare()
psutil.cpu_percent(0, percpu=True)
psutil.cpu_percent(0, percpu = True)
# Give it some time
time.sleep(1)
# Retrieve info for system
info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True))
info["system"]["cpu_max"] = 100.0 * psutil.cpu_count()
info["system"]["procs"] = len(psutil.pids())
meminfo = psutil.virtual_memory()
info["system"]["cpu_max"] = 100.0 * psutil.NUM_CPUS
info["system"]["procs"] = len(psutil.get_pid_list())
# psutil > 0.6.0's psutil.virtual_memory() would be better here,
# but this should give the same info.
meminfo = psutil.phymem_usage()
info["system"]["mem_total"] = meminfo.total
info["system"]["mem_used"] = meminfo.used
info["system"]["mem_used"] = int(meminfo.total * meminfo.percent / 100)
# Retrieve info for each PID
for pid in self:

72
nilmrun/server.py
View File

@ -1,12 +1,19 @@
"""CherryPy-based server for running NILM filters via HTTP"""
import cherrypy
import sys
import os
import socket
import simplejson as json
import traceback
import time
from nilmdb.utils.printf import sprintf
import nilmdb
from nilmdb.utils.printf import *
from nilmdb.server.serverutil import (
chunked_response,
response_type,
workaround_cp_bug_1200,
exception_to_httperror,
CORS_allow,
json_to_request_params,
@ -17,12 +24,11 @@ from nilmdb.server.serverutil import (
)
from nilmdb.utils import serializer_proxy
import nilmrun
import nilmrun.processmanager
import nilmrun.testfilter
# Add CORS_allow tool
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
# CherryPy apps
class App(object):
"""Root application for NILM runner"""
@ -49,7 +55,6 @@ class App(object):
def version(self):
return nilmrun.__version__
class AppProcess(object):
def __init__(self, manager):
@ -71,7 +76,7 @@ class AppProcess(object):
# /process/status
@cherrypy.expose
@cherrypy.tools.json_out()
def status(self, pid, clear=False):
def status(self, pid, clear = False):
"""Return status about a process. If clear = True, also clear
the log."""
clear = bool_param(clear)
@ -101,7 +106,7 @@ class AppProcess(object):
@cherrypy.expose
@cherrypy.tools.json_in()
@cherrypy.tools.json_out()
@cherrypy.tools.CORS_allow(methods=["POST"])
@cherrypy.tools.CORS_allow(methods = ["POST"])
def remove(self, pid):
"""Remove a process from the manager, killing it if necessary."""
if pid not in self.manager:
@ -113,7 +118,6 @@ class AppProcess(object):
self.manager.remove(pid)
return status
class AppRun(object):
def __init__(self, manager):
self.manager = manager
@ -123,7 +127,7 @@ class AppRun(object):
@cherrypy.tools.json_in()
@cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods=["POST"])
@cherrypy.tools.CORS_allow(methods = ["POST"])
def command(self, argv):
"""Execute an arbitrary program on the server. argv is a
list of the program and its arguments: 'argv[0]' is the program
@ -138,8 +142,8 @@ class AppRun(object):
@cherrypy.tools.json_in()
@cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods=["POST"])
def code(self, code, args=None):
@cherrypy.tools.CORS_allow(methods = ["POST"])
def code(self, code, args = None):
"""Execute arbitrary Python code. 'code' is a formatted string.
It will be run as if it were written into a Python file and
executed. 'args' is a list of strings, and they are passed
@ -152,21 +156,23 @@ class AppRun(object):
"args must be a list of strings")
return self.manager.run_code(code, args)
class Server(object):
def __init__(self, host='127.0.0.1', port=8080,
force_traceback=False, # include traceback in all errors
basepath='', # base URL path for cherrypy.tree
def __init__(self, host = '127.0.0.1', port = 8080,
embedded = True, # hide diagnostics and output, etc
force_traceback = False, # include traceback in all errors
basepath = '', # base URL path for cherrypy.tree
):
self.embedded = embedded
# Build up global server configuration
cherrypy.config.update({
'environment': 'embedded',
'server.socket_host': host,
'server.socket_port': port,
'engine.autoreload_on': False,
'server.max_request_body_size': 8*1024*1024,
})
if self.embedded:
cherrypy.config.update({ 'environment': 'embedded' })
# Build up application specific configuration
app_config = {}
@ -175,25 +181,23 @@ class Server(object):
})
# Some default headers to just help identify that things are working
app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
# Set up Cross-Origin Resource Sharing (CORS) handler so we
# can correctly respond to browsers' CORS preflight requests.
# This also limits verbs to GET and HEAD by default.
app_config.update({
'tools.CORS_allow.on': True,
'tools.CORS_allow.methods': ['GET', 'HEAD']
})
app_config.update({ 'tools.CORS_allow.on': True,
'tools.CORS_allow.methods': ['GET', 'HEAD'] })
# Configure the 'json_in' tool to also allow other content-types
# (like x-www-form-urlencoded), and to treat JSON as a dict that
# fills requests.param.
app_config.update({'tools.json_in.force': False,
'tools.json_in.processor': json_to_request_params})
app_config.update({ 'tools.json_in.force': False,
'tools.json_in.processor': json_to_request_params })
# Send tracebacks in error responses. They're hidden by the
# error_page function for client errors (code 400-499).
app_config.update({'request.show_tracebacks': True})
app_config.update({ 'request.show_tracebacks' : True })
self.force_traceback = force_traceback
# Patch CherryPy error handler to never pad out error messages.
@ -206,12 +210,11 @@ class Server(object):
manager = serializer_proxy(nilmrun.processmanager.ProcessManager)()
# Build up the application and mount it
self._manager = manager
root = App()
root.process = AppProcess(manager)
root.run = AppRun(manager)
cherrypy.tree.apps = {}
cherrypy.tree.mount(root, basepath, config={"/": app_config})
cherrypy.tree.mount(root, basepath, config = { "/" : app_config })
# Set up the WSGI application pointer for external programs
self.wsgi_application = cherrypy.tree
@ -221,19 +224,16 @@ class Server(object):
return json_error_page(status, message, traceback, version,
self.force_traceback)
def start(self, blocking=False, event=None):
cherrypy_start(blocking, event)
def start(self, blocking = False, event = None):
cherrypy_start(blocking, event, self.embedded)
def stop(self):
cherrypy_stop()
# Multiple processes and threads should be OK here, but we'll still
# follow the NilmDB approach of having just one globally initialized
# copy of the server object.
_wsgi_server = None
def wsgi_application(basepath): # pragma: no cover
"""Return a WSGI application object.
@ -247,11 +247,13 @@ def wsgi_application(basepath): # pragma: no cover
# Try to start the server
try:
_wsgi_server = nilmrun.server.Server(
basepath=basepath.rstrip('/'))
embedded = True,
basepath = basepath.rstrip('/'))
except Exception:
# Build an error message on failure
import pprint
err = "Initializing nilmrun failed:\n\n"
err = sprintf("Initializing nilmrun failed:\n\n",
dbpath)
err += traceback.format_exc()
try:
import pwd
@ -266,10 +268,8 @@ def wsgi_application(basepath): # pragma: no cover
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
if _wsgi_server is None:
# Serve up the error with our own mini WSGI app.
headers = [
('Content-type', 'text/plain'),
('Content-length', str(len(err)))
]
headers = [ ('Content-type', 'text/plain'),
('Content-length', str(len(err))) ]
start_response("500 Internal Server Error", headers)
return [err]

22
nilmrun/testfilter.py Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/python
from nilmdb.utils.printf import *
import time
import signal
import sys
# This is just for testing the process management.
def test(n):
n = int(n)
if n < 0: # raise an exception
raise Exception("test exception")
if n == 0: # ignore SIGTERM and count to 100
n = 100
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for x in range(n):
s = sprintf("dummy %d\n", x)
if x & 1:
sys.stdout.write(s)
else:
sys.stderr.write(s)
time.sleep(0.1)

2
requirements.txt
View File

@ -1,2 +0,0 @@
nilmdb>=2.0.3
psutil==5.7.2

51
scripts/kill.py
View File

@ -1,51 +0,0 @@
#!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
import nilmrun
import argparse
import os
import sys
def main():
"""Kill/remove a process from the NilmRun server"""
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'Kill/remove a process from the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)
group.add_argument('-n', '--noverify', action="store_true",
help = 'Disable SSL certificate verification')
group = parser.add_argument_group("Program")
group.add_argument('-q', '--quiet', action="store_true",
help = "Don't print out the final log contents")
group.add_argument('pid', nargs='+', help="PIDs to kill")
args = parser.parse_args()
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
# Kill or remove process
all_failed = True
for pid in args.pid:
try:
s = client.post("process/remove", { "pid": pid })
if not args.quiet:
sys.stdout.write(s['log'])
all_failed = False
except ClientError as e:
if "404" in e.status:
fprintf(sys.stderr, "no such pid: %s\n", pid)
else:
raise
# Return error if we failed to remove any of them
if all_failed:
raise SystemExit(1)
if __name__ == "__main__":
main()

20
scripts/nilmrun_server.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
import nilmrun.server
import argparse
@ -11,8 +11,9 @@ def main():
parser = argparse.ArgumentParser(
description = 'Run the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
parser.add_argument("-V", "--version", action="version",
version = nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-a', '--address',
@ -35,24 +36,25 @@ def main():
embedded = False
server = nilmrun.server.Server(host = args.address,
port = args.port,
embedded = embedded,
force_traceback = args.traceback)
# Print info
if not args.quiet:
print("NilmRun version: %s" % nilmrun.__version__)
print(("Note: This server does not do any authentication! " +
"Anyone who can connect can run arbitrary commands."))
print "NilmRun version: %s" % nilmrun.__version__
print ("Note: This server does not do any authentication! " +
"Anyone who can connect can run arbitrary commands.")
if args.address == '0.0.0.0' or args.address == '::':
host = socket.getfqdn()
else:
host = args.address
print("Server URL: http://%s:%d/" % ( host, args.port))
print("----")
print "Server URL: http://%s:%d/" % ( host, args.port)
print "----"
server.start(blocking = True)
if not args.quiet:
print("Shutting down")
print "Shutting down"
if __name__ == "__main__":
main()

67
scripts/ps.py
View File

@ -1,67 +0,0 @@
#!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
import datetime_tz
import nilmrun
import argparse
import os
def main():
"""List NilmRun processes"""
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'List NilmRun processes',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)
group.add_argument('-n', '--noverify', action="store_true",
help = 'Disable SSL certificate verification')
args = parser.parse_args()
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
# Print overall system info
info = client.get("process/info")
total = info['total']
system = info['system']
printf(" procs: %d nilm, %d other\n", info['total']['procs'],
info['system']['procs'] - info['total']['procs'])
printf(" cpu: %d%% nilm, %d%% other, %d%% max\n",
round(info['total']['cpu_percent']),
round(info['system']['cpu_percent'] - info['total']['cpu_percent']),
round(info['system']['cpu_max']))
printf(" mem: %d MiB used, %d MiB total, %d%%\n",
round(info['system']['mem_used'] / 1048576.0),
round(info['system']['mem_total'] / 1048576.0),
round(info['system']['mem_used'] * 100.0
/ info['system']['mem_total']))
# Print process detail for each managed process
fmt = "%-36s %-6s %-15s %-4s %-3s %-5s\n"
printf(fmt, "PID", "STATE", "SINCE", "PROC", "CPU", "LOG")
if len(info['pids']) == 0:
printf("No running processes\n")
raise SystemExit(0)
for pid in sorted(info['pids'].keys()):
pidinfo = client.get("process/status", { "pid": pid })
if pidinfo['alive']:
status = "alive"
else:
if pidinfo['exitcode']:
status = "error"
else:
status = "done"
dt = datetime_tz.datetime_tz.fromtimestamp(pidinfo['start_time'])
since = dt.strftime("%m/%d-%H:%M:%S")
printf(fmt, pid, status, since, info['pids'][pid]['procs'],
str(int(round(info['pids'][pid]['cpu_percent']))),
len(pidinfo['log']))
if __name__ == "__main__":
main()

61
scripts/run.py
View File

@ -1,61 +0,0 @@
#!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
import nilmrun
import argparse
import os
import time
import sys
def main():
"""Run a command on the NilmRun server"""
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'Run a command on the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)
group.add_argument('-n', '--noverify', action="store_true",
help = 'Disable SSL certificate verification')
group = parser.add_argument_group("Program")
group.add_argument('-d', '--detach', action="store_true",
help = 'Run process and return immediately without '
'printing its output')
group.add_argument('cmd', help="Command to run")
group.add_argument('arg', nargs=argparse.REMAINDER,
help="Arguments for command")
args = parser.parse_args()
client = HTTPClient(baseurl=args.url, verify_ssl=not args.noverify,
post_json=True)
# Run command
pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })
# If we're detaching, just print the PID
if args.detach:
print(pid)
raise SystemExit(0)
# Otherwise, watch the log output, and kill the process when it's done
# or when this script terminates.
try:
while True:
s = client.get("process/status", { "pid": pid, "clear": 1 })
sys.stdout.write(s['log'])
sys.stdout.flush()
if not s['alive']:
break
time.sleep(1)
finally:
s = client.post("process/remove", { "pid": pid })
raise SystemExit(s['exitcode'])
if __name__ == "__main__":
main()

16
setup.cfg
View File

@ -20,19 +20,3 @@ cover-erase=1
stop=1
verbosity=2
tests=tests
[versioneer]
VCS=git
style=pep440
versionfile_source=nilmrun/_version.py
versionfile_build=nilmrun/_version.py
tag_prefix=nilmrun-
parentdir_prefix=nilmrun-
[flake8]
exclude=_version.py
extend-ignore=E731
[pylint]
ignore=_version.py
disable=C0103,C0111,R0913,R0914

56
setup.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
# To release a new version, tag it:
# git tag -a nilmrun-1.1 -m "Version 1.1"
@ -6,31 +6,66 @@
# Then just package it up:
# python setup.py sdist
# This is supposed to be using Distribute:
#
# distutils provides a "setup" method.
# setuptools is a set of monkeypatches on top of that.
# distribute is a particular version/implementation of setuptools.
#
# So we don't really know if this is using the old setuptools or the
# Distribute-provided version of setuptools.
import traceback
import sys
import os
from setuptools import setup
try:
from setuptools import setup, find_packages
import distutils.version
except ImportError:
traceback.print_exc()
print "Please install the prerequisites listed in README.txt"
sys.exit(1)
# Versioneer manages version numbers from git tags.
# https://github.com/warner/python-versioneer
import versioneer
versioneer.versionfile_source = 'nilmrun/_version.py'
versioneer.versionfile_build = 'nilmrun/_version.py'
versioneer.tag_prefix = 'nilmrun-'
versioneer.parentdir_prefix = 'nilmrun-'
# Get list of requirements to use in `install_requires` below. Note
# that we don't make a distinction between things that are actually
# required for end-users vs developers (or use `test_requires` or
# anything else) -- just install everything for simplicity.
install_requires = open('requirements.txt').readlines()
# Hack to workaround logging/multiprocessing issue:
# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
try: import multiprocessing
except: pass
# We need a MANIFEST.in. Generate it here rather than polluting the
# repository with yet another setup-related file.
with open("MANIFEST.in", "w") as m:
m.write("""
# Root
include README.txt
include setup.py
include versioneer.py
include Makefile
""")
# Run setup
setup(name='nilmrun',
version = versioneer.get_version(),
cmdclass = versioneer.get_cmdclass(),
url = 'https://git.jim.sh/nilm/nilmrun.git',
url = 'https://git.jim.sh/jim/lees/nilmrun.git',
author = 'Jim Paris',
description = "NILM Database Filter Runner",
long_description = "NILM Database Filter Runner",
license = "Proprietary",
author_email = 'jim@jtan.com',
install_requires = install_requires,
install_requires = [ 'nilmdb >= 1.8.3',
'psutil >= 0.3.0',
'cherrypy >= 3.2',
'simplejson',
],
packages = [ 'nilmrun',
'nilmrun.scripts',
],
@ -40,9 +75,6 @@ setup(name='nilmrun',
entry_points = {
'console_scripts': [
'nilmrun-server = nilmrun.scripts.nilmrun_server:main',
'nilmrun-ps = nilmrun.scripts.ps:main',
'nilmrun-run = nilmrun.scripts.run:main',
'nilmrun-kill = nilmrun.scripts.kill:main',
],
},
zip_safe = False,

2
tests/runtests.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
import nose
import os

86
tests/test_nilmrun.py
View File

@ -15,15 +15,14 @@ import distutils.version
import os
import sys
import threading
import io
import json
import cStringIO
import simplejson as json
import unittest
import warnings
import time
import re
import urllib.request, urllib.error, urllib.parse
from urllib.request import urlopen
from urllib.error import HTTPError
import urllib2
from urllib2 import urlopen, HTTPError
import requests
import pprint
import textwrap
@ -129,27 +128,10 @@ class TestClient(object):
def _run_testfilter(self, client, args):
code = textwrap.dedent("""
from nilmdb.utils.printf import *
import time
import signal
import json
import nilmrun.testfilter
import simplejson as json
import sys
# This is just for testing the process management.
def test(n):
n = int(n)
if n < 0: # raise an exception
raise Exception("test exception")
if n == 0: # ignore SIGTERM and count to 100
n = 100
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for x in range(n):
s = sprintf("dummy %d\\n", x)
if x & 1:
sys.stdout.write(s)
else:
sys.stderr.write(s)
time.sleep(0.1)
test(json.loads(sys.argv[1]))
nilmrun.testfilter.test(json.loads(sys.argv[1]))
""")
jsonargs = json.dumps(args)
return client.post("run/code", { "code": code, "args": [ jsonargs ] })
@ -287,9 +269,9 @@ class TestClient(object):
# basic code snippet
code = textwrap.dedent("""
print('hello')
print 'hello'
def foo(arg):
print('world')
print 'world'
""")
status = do(code, [], False)
eq_("hello\n", status["log"])
@ -298,7 +280,7 @@ class TestClient(object):
# compile error
code = textwrap.dedent("""
def foo(arg:
print('hello')
print 'hello'
""")
status = do(code, [], False)
in_("SyntaxError", status["log"])
@ -323,7 +305,7 @@ class TestClient(object):
# argument handling (strings come in as unicode)
code = textwrap.dedent("""
import sys
print(sys.argv[1], sys.argv[2])
print sys.argv[1].encode('ascii'), sys.argv[2]
sys.exit(0) # also test raising SystemExit
""")
with assert_raises(ClientError) as e:
@ -336,9 +318,9 @@ class TestClient(object):
# try killing a long-running process
code = textwrap.dedent("""
import time
print('hello')
print 'hello'
time.sleep(60)
print('world')
print 'world'
""")
status = do(code, [], True)
eq_(status["log"], "hello\n")
@ -347,7 +329,7 @@ class TestClient(object):
# default arguments are empty
code = textwrap.dedent("""
import sys
print('args:', len(sys.argv[1:]))
print 'args:', len(sys.argv[1:])
""")
status = do(code, None, False)
eq_(status["log"], "args: 0\n")
@ -370,10 +352,8 @@ class TestClient(object):
# start some processes
a = client.post("run/command", { "argv": ["sleep","60"] } )
b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } )
c = client.post("run/command", { "argv": [
"sh","-c","dd if=/dev/zero of=/dev/null;true"] } )
d = client.post("run/command", { "argv": [
"dd", "if=/dev/zero", "of=/dev/null" ] } )
c = client.post("run/command", { "argv": ["sh","-c","burnP5;true"] } )
d = client.post("run/command", { "argv": ["burnP5" ] } )
info = client.get("process/info")
eq_(info["pids"][a]["procs"], 1)
@ -385,13 +365,9 @@ class TestClient(object):
lt_(20, info["pids"][c]["cpu_percent"])
lt_(80, info["system"]["cpu_percent"])
for x in range(10):
time.sleep(1)
time.sleep(2)
info = client.get("process/info")
if info["pids"][b]["procs"] != 2:
break
else:
raise Exception("process B didn't die: " + str(info["pids"][b]))
eq_(info["pids"][b]["procs"], 0)
# kill all processes
for pid in client.get("process/list"):
@ -401,34 +377,16 @@ class TestClient(object):
client = HTTPClient(baseurl = testurl, post_json = True)
eq_(client.get("process/list"), [])
def verify(cmd, result):
pid = client.post("run/command", { "argv": [
"/bin/bash", "-c", cmd ] })
pid = client.post("run/command", { "argv": [ "sh", "-c", cmd ] })
eq_(client.get("process/list"), [pid])
status = self.wait_end(client, pid)
eq_(result, status["log"])
# Unicode should work
verify("echo -n hello", "hello")
verify("echo -n ☠", "☠")
verify("echo -ne \\\\xe2\\\\x98\\\\xa0", "☠")
verify(u"echo -n ☠", u"☠")
verify("echo -ne \\\\xe2\\\\x98\\\\xa0", u"☠")
# Programs that spit out invalid UTF-8 should get replacement
# markers
verify("echo -ne \\\\xae", "\ufffd")
def test_client_11_atexit(self):
# Leave a directory and running process behind, for the atexit
# handler to clean up. Here we trigger the atexit manually,
# since it's hard to trigger it as part of the test suite.
client = HTTPClient(baseurl = testurl, post_json = True)
code = textwrap.dedent("""
import time
time.sleep(10)
""")
client.post("run/code", { "code": code, "args": [ "hello"] })
# Trigger atexit function
test_server._manager._atexit()
# Ensure no processes exit
eq_(client.get("process/list"), [])
verify("echo -ne \\\\xae", u"\ufffd")

2
tests/testutil/helpers.py
View File

@ -1,7 +1,7 @@
# Just some helpers for test functions
def myrepr(x):
if isinstance(x, str):
if isinstance(x, basestring):
return '"' + x + '"'
else:
return repr(x)

2103
versioneer.py
View File

File diff suppressed because it is too large Load Diff