Compare commits
20 Commits
nilmrun-1.
...
master
Author | SHA1 | Date | |
---|---|---|---|
eae6dd623f | |||
09a9ed9734 | |||
079a2b5192 | |||
e7f52a4013 | |||
c36b9b97e0 | |||
549a27e66c | |||
cd68389e9a | |||
6faf563bda | |||
58fd9d1559 | |||
2bc939d42d | |||
fe36722684 | |||
24740a838e | |||
d332fa1e0f | |||
cbc7a7dd55 | |||
38c3e67cf9 | |||
7f05a0fb62 | |||
81c2ad07d4 | |||
3588e843ac | |||
9309fd9b57 | |||
21bd1bd050 |
2
.gitattributes
vendored
2
.gitattributes
vendored
|
@ -1 +1 @@
|
||||||
src/_version.py export-subst
|
nilmrun/_version.py export-subst
|
||||||
|
|
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -3,8 +3,8 @@ build/
|
||||||
*.pyc
|
*.pyc
|
||||||
dist/
|
dist/
|
||||||
nilmrun.egg-info/
|
nilmrun.egg-info/
|
||||||
|
.eggs/
|
||||||
|
|
||||||
# This gets generated as needed by setup.py
|
# This gets generated as needed by setup.py
|
||||||
MANIFEST.in
|
|
||||||
MANIFEST
|
MANIFEST
|
||||||
|
|
||||||
|
|
8
MANIFEST.in
Normal file
8
MANIFEST.in
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
# Root
|
||||||
|
include README.md
|
||||||
|
include setup.py
|
||||||
|
include versioneer.py
|
||||||
|
include Makefile
|
||||||
|
|
||||||
|
# Version
|
||||||
|
include nilmrun/_version.py
|
33
Makefile
33
Makefile
|
@ -1,48 +1,49 @@
|
||||||
# By default, run the tests.
|
# By default, run the tests.
|
||||||
all: test
|
all: test
|
||||||
|
|
||||||
test2:
|
|
||||||
nilmrun/trainola.py data.js
|
|
||||||
|
|
||||||
version:
|
version:
|
||||||
python setup.py version
|
python3 setup.py version
|
||||||
|
|
||||||
build:
|
build:
|
||||||
python setup.py build_ext --inplace
|
python3 setup.py build_ext --inplace
|
||||||
|
|
||||||
dist: sdist
|
dist: sdist
|
||||||
sdist:
|
sdist:
|
||||||
python setup.py sdist
|
python3 setup.py sdist
|
||||||
|
|
||||||
install:
|
install:
|
||||||
python setup.py install
|
python3 setup.py install
|
||||||
|
|
||||||
develop:
|
develop:
|
||||||
python setup.py develop
|
python3 setup.py develop
|
||||||
|
|
||||||
docs:
|
docs:
|
||||||
make -C docs
|
make -C docs
|
||||||
|
|
||||||
|
ctrl: flake
|
||||||
|
flake:
|
||||||
|
flake8 nilmrun
|
||||||
lint:
|
lint:
|
||||||
pylint --rcfile=.pylintrc nilmdb
|
pylint3 --rcfile=setup.cfg nilmrun
|
||||||
|
|
||||||
test:
|
test:
|
||||||
ifeq ($(INSIDE_EMACS), t)
|
ifneq ($(INSIDE_EMACS),)
|
||||||
# Use the slightly more flexible script
|
# Use the slightly more flexible script
|
||||||
python setup.py build_ext --inplace
|
python3 setup.py build_ext --inplace
|
||||||
python tests/runtests.py
|
python3 tests/runtests.py
|
||||||
else
|
else
|
||||||
# Let setup.py check dependencies, build stuff, and run the test
|
# Let setup.py check dependencies, build stuff, and run the test
|
||||||
python setup.py nosetests
|
python3 setup.py nosetests
|
||||||
endif
|
endif
|
||||||
|
|
||||||
clean::
|
clean::
|
||||||
|
find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
|
||||||
rm -f .coverage
|
rm -f .coverage
|
||||||
find . -name '*pyc' | xargs rm -f
|
rm -rf nilmrun.egg-info/ build/
|
||||||
rm -rf nilmtools.egg-info/ build/ MANIFEST.in
|
|
||||||
make -C docs clean
|
make -C docs clean
|
||||||
|
|
||||||
gitclean::
|
gitclean::
|
||||||
git clean -dXf
|
git clean -dXf
|
||||||
|
|
||||||
.PHONY: all version dist sdist install docs lint test clean gitclean
|
.PHONY: all version dist sdist install docs test
|
||||||
|
.PHONY: ctrl lint flake clean gitclean
|
||||||
|
|
32
README.md
Normal file
32
README.md
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
# nilmrun: Run NilmDB filters
|
||||||
|
by Jim Paris <jim@jtan.com>
|
||||||
|
|
||||||
|
## Prerequisites:
|
||||||
|
|
||||||
|
# Runtime and build environments
|
||||||
|
sudo apt-get install python3
|
||||||
|
|
||||||
|
# Create a new Python virtual environment to isolate deps.
|
||||||
|
python3 -m venv ../venv
|
||||||
|
source ../venv/bin/activate # run "deactivate" to leave
|
||||||
|
|
||||||
|
# Install all Python dependencies
|
||||||
|
pip3 install -r requirements.txt
|
||||||
|
|
||||||
|
## Install:
|
||||||
|
|
||||||
|
Install it into the virtual environment
|
||||||
|
|
||||||
|
python3 setup.py install
|
||||||
|
|
||||||
|
If you want to instead install it system-wide, you will also need to
|
||||||
|
install the requirements system-wide:
|
||||||
|
|
||||||
|
sudo pip3 install -r requirements.txt
|
||||||
|
sudo python3 setup.py install
|
||||||
|
|
||||||
|
## Usage:
|
||||||
|
|
||||||
|
nilmrun-server --help
|
||||||
|
|
||||||
|
See docs/wsgi.md for info on setting up a WSGI application in Apache.
|
20
README.txt
20
README.txt
|
@ -1,20 +0,0 @@
|
||||||
nilmrun: Run NilmDB filters
|
|
||||||
by Jim Paris <jim@jtan.com>
|
|
||||||
|
|
||||||
Prerequisites:
|
|
||||||
|
|
||||||
# Runtime and build environments
|
|
||||||
sudo apt-get install python2.7 python-setuptools
|
|
||||||
|
|
||||||
# Plus nilmdb and its dependencies
|
|
||||||
nilmdb (1.9.5+)
|
|
||||||
|
|
||||||
Install:
|
|
||||||
|
|
||||||
python setup.py install
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
|
|
||||||
nilmrun-server --help
|
|
||||||
|
|
||||||
See docs/wsgi.md for info on setting up a WSGI application in Apache.
|
|
|
@ -1,5 +1,3 @@
|
||||||
import nilmrun.processmanager
|
|
||||||
|
|
||||||
from ._version import get_versions
|
from ._version import get_versions
|
||||||
__version__ = get_versions()['version']
|
__version__ = get_versions()['version']
|
||||||
del get_versions
|
del get_versions
|
||||||
|
|
|
@ -1,197 +1,520 @@
|
||||||
|
|
||||||
IN_LONG_VERSION_PY = True
|
|
||||||
# This file helps to compute a version number in source trees obtained from
|
# This file helps to compute a version number in source trees obtained from
|
||||||
# git-archive tarball (such as those provided by githubs download-from-tag
|
# git-archive tarball (such as those provided by githubs download-from-tag
|
||||||
# feature). Distribution tarballs (build by setup.py sdist) and build
|
# feature). Distribution tarballs (built by setup.py sdist) and build
|
||||||
# directories (produced by setup.py build) will contain a much shorter file
|
# directories (produced by setup.py build) will contain a much shorter file
|
||||||
# that just contains the computed version number.
|
# that just contains the computed version number.
|
||||||
|
|
||||||
# This file is released into the public domain. Generated by
|
# This file is released into the public domain. Generated by
|
||||||
# versioneer-0.7+ (https://github.com/warner/python-versioneer)
|
# versioneer-0.18 (https://github.com/warner/python-versioneer)
|
||||||
|
|
||||||
# these strings will be replaced by git during git-archive
|
|
||||||
git_refnames = "$Format:%d$"
|
|
||||||
git_full = "$Format:%H$"
|
|
||||||
|
|
||||||
|
"""Git implementation of _version.py."""
|
||||||
|
|
||||||
|
import errno
|
||||||
|
import os
|
||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
def run_command(args, cwd=None, verbose=False):
|
|
||||||
try:
|
def get_keywords():
|
||||||
# remember shell=False, so use git.cmd on windows, not just git
|
"""Get the keywords needed to look up the version information."""
|
||||||
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
|
# these strings will be replaced by git during git-archive.
|
||||||
except EnvironmentError:
|
# setup.py/versioneer.py will grep for the variable names, so they must
|
||||||
e = sys.exc_info()[1]
|
# each be defined on a line of their own. _version.py will just call
|
||||||
|
# get_keywords().
|
||||||
|
git_refnames = "$Format:%d$"
|
||||||
|
git_full = "$Format:%H$"
|
||||||
|
git_date = "$Format:%ci$"
|
||||||
|
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
|
||||||
|
return keywords
|
||||||
|
|
||||||
|
|
||||||
|
class VersioneerConfig:
|
||||||
|
"""Container for Versioneer configuration parameters."""
|
||||||
|
|
||||||
|
|
||||||
|
def get_config():
|
||||||
|
"""Create, populate and return the VersioneerConfig() object."""
|
||||||
|
# these strings are filled in when 'setup.py versioneer' creates
|
||||||
|
# _version.py
|
||||||
|
cfg = VersioneerConfig()
|
||||||
|
cfg.VCS = "git"
|
||||||
|
cfg.style = "pep440"
|
||||||
|
cfg.tag_prefix = "nilmrun-"
|
||||||
|
cfg.parentdir_prefix = "nilmrun-"
|
||||||
|
cfg.versionfile_source = "nilmrun/_version.py"
|
||||||
|
cfg.verbose = False
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
class NotThisMethod(Exception):
|
||||||
|
"""Exception raised if a method is not valid for the current scenario."""
|
||||||
|
|
||||||
|
|
||||||
|
LONG_VERSION_PY = {}
|
||||||
|
HANDLERS = {}
|
||||||
|
|
||||||
|
|
||||||
|
def register_vcs_handler(vcs, method): # decorator
|
||||||
|
"""Decorator to mark a method as the handler for a particular VCS."""
|
||||||
|
def decorate(f):
|
||||||
|
"""Store f in HANDLERS[vcs][method]."""
|
||||||
|
if vcs not in HANDLERS:
|
||||||
|
HANDLERS[vcs] = {}
|
||||||
|
HANDLERS[vcs][method] = f
|
||||||
|
return f
|
||||||
|
return decorate
|
||||||
|
|
||||||
|
|
||||||
|
def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
|
||||||
|
env=None):
|
||||||
|
"""Call the given command(s)."""
|
||||||
|
assert isinstance(commands, list)
|
||||||
|
p = None
|
||||||
|
for c in commands:
|
||||||
|
try:
|
||||||
|
dispcmd = str([c] + args)
|
||||||
|
# remember shell=False, so use git.cmd on windows, not just git
|
||||||
|
p = subprocess.Popen([c] + args, cwd=cwd, env=env,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=(subprocess.PIPE if hide_stderr
|
||||||
|
else None))
|
||||||
|
break
|
||||||
|
except EnvironmentError:
|
||||||
|
e = sys.exc_info()[1]
|
||||||
|
if e.errno == errno.ENOENT:
|
||||||
|
continue
|
||||||
|
if verbose:
|
||||||
|
print("unable to run %s" % dispcmd)
|
||||||
|
print(e)
|
||||||
|
return None, None
|
||||||
|
else:
|
||||||
if verbose:
|
if verbose:
|
||||||
print("unable to run %s" % args[0])
|
print("unable to find command, tried %s" % (commands,))
|
||||||
print(e)
|
return None, None
|
||||||
return None
|
|
||||||
stdout = p.communicate()[0].strip()
|
stdout = p.communicate()[0].strip()
|
||||||
if sys.version >= '3':
|
if sys.version_info[0] >= 3:
|
||||||
stdout = stdout.decode()
|
stdout = stdout.decode()
|
||||||
if p.returncode != 0:
|
if p.returncode != 0:
|
||||||
if verbose:
|
if verbose:
|
||||||
print("unable to run %s (error)" % args[0])
|
print("unable to run %s (error)" % dispcmd)
|
||||||
return None
|
print("stdout was %s" % stdout)
|
||||||
return stdout
|
return None, p.returncode
|
||||||
|
return stdout, p.returncode
|
||||||
|
|
||||||
|
|
||||||
import sys
|
def versions_from_parentdir(parentdir_prefix, root, verbose):
|
||||||
import re
|
"""Try to determine the version from the parent directory name.
|
||||||
import os.path
|
|
||||||
|
|
||||||
def get_expanded_variables(versionfile_source):
|
Source tarballs conventionally unpack into a directory that includes both
|
||||||
|
the project name and a version string. We will also support searching up
|
||||||
|
two directory levels for an appropriately named parent directory
|
||||||
|
"""
|
||||||
|
rootdirs = []
|
||||||
|
|
||||||
|
for i in range(3):
|
||||||
|
dirname = os.path.basename(root)
|
||||||
|
if dirname.startswith(parentdir_prefix):
|
||||||
|
return {"version": dirname[len(parentdir_prefix):],
|
||||||
|
"full-revisionid": None,
|
||||||
|
"dirty": False, "error": None, "date": None}
|
||||||
|
else:
|
||||||
|
rootdirs.append(root)
|
||||||
|
root = os.path.dirname(root) # up a level
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
print("Tried directories %s but none started with prefix %s" %
|
||||||
|
(str(rootdirs), parentdir_prefix))
|
||||||
|
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
|
||||||
|
|
||||||
|
|
||||||
|
@register_vcs_handler("git", "get_keywords")
|
||||||
|
def git_get_keywords(versionfile_abs):
|
||||||
|
"""Extract version information from the given file."""
|
||||||
# the code embedded in _version.py can just fetch the value of these
|
# the code embedded in _version.py can just fetch the value of these
|
||||||
# variables. When used from setup.py, we don't want to import
|
# keywords. When used from setup.py, we don't want to import _version.py,
|
||||||
# _version.py, so we do it with a regexp instead. This function is not
|
# so we do it with a regexp instead. This function is not used from
|
||||||
# used from _version.py.
|
# _version.py.
|
||||||
variables = {}
|
keywords = {}
|
||||||
try:
|
try:
|
||||||
for line in open(versionfile_source,"r").readlines():
|
f = open(versionfile_abs, "r")
|
||||||
|
for line in f.readlines():
|
||||||
if line.strip().startswith("git_refnames ="):
|
if line.strip().startswith("git_refnames ="):
|
||||||
mo = re.search(r'=\s*"(.*)"', line)
|
mo = re.search(r'=\s*"(.*)"', line)
|
||||||
if mo:
|
if mo:
|
||||||
variables["refnames"] = mo.group(1)
|
keywords["refnames"] = mo.group(1)
|
||||||
if line.strip().startswith("git_full ="):
|
if line.strip().startswith("git_full ="):
|
||||||
mo = re.search(r'=\s*"(.*)"', line)
|
mo = re.search(r'=\s*"(.*)"', line)
|
||||||
if mo:
|
if mo:
|
||||||
variables["full"] = mo.group(1)
|
keywords["full"] = mo.group(1)
|
||||||
|
if line.strip().startswith("git_date ="):
|
||||||
|
mo = re.search(r'=\s*"(.*)"', line)
|
||||||
|
if mo:
|
||||||
|
keywords["date"] = mo.group(1)
|
||||||
|
f.close()
|
||||||
except EnvironmentError:
|
except EnvironmentError:
|
||||||
pass
|
pass
|
||||||
return variables
|
return keywords
|
||||||
|
|
||||||
def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
|
|
||||||
refnames = variables["refnames"].strip()
|
@register_vcs_handler("git", "keywords")
|
||||||
|
def git_versions_from_keywords(keywords, tag_prefix, verbose):
|
||||||
|
"""Get version information from git keywords."""
|
||||||
|
if not keywords:
|
||||||
|
raise NotThisMethod("no keywords at all, weird")
|
||||||
|
date = keywords.get("date")
|
||||||
|
if date is not None:
|
||||||
|
# git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
|
||||||
|
# datestamp. However we prefer "%ci" (which expands to an "ISO-8601
|
||||||
|
# -like" string, which we must then edit to make compliant), because
|
||||||
|
# it's been around since git-1.5.3, and it's too difficult to
|
||||||
|
# discover which version we're using, or to work around using an
|
||||||
|
# older one.
|
||||||
|
date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
|
||||||
|
refnames = keywords["refnames"].strip()
|
||||||
if refnames.startswith("$Format"):
|
if refnames.startswith("$Format"):
|
||||||
if verbose:
|
if verbose:
|
||||||
print("variables are unexpanded, not using")
|
print("keywords are unexpanded, not using")
|
||||||
return {} # unexpanded, so not in an unpacked git-archive tarball
|
raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
|
||||||
refs = set([r.strip() for r in refnames.strip("()").split(",")])
|
refs = set([r.strip() for r in refnames.strip("()").split(",")])
|
||||||
for ref in list(refs):
|
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
|
||||||
if not re.search(r'\d', ref):
|
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
|
||||||
if verbose:
|
TAG = "tag: "
|
||||||
print("discarding '%s', no digits" % ref)
|
tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
|
||||||
refs.discard(ref)
|
if not tags:
|
||||||
# Assume all version tags have a digit. git's %d expansion
|
# Either we're using git < 1.8.3, or there really are no tags. We use
|
||||||
# behaves like git log --decorate=short and strips out the
|
# a heuristic: assume all version tags have a digit. The old git %d
|
||||||
# refs/heads/ and refs/tags/ prefixes that would let us
|
# expansion behaves like git log --decorate=short and strips out the
|
||||||
# distinguish between branches and tags. By ignoring refnames
|
# refs/heads/ and refs/tags/ prefixes that would let us distinguish
|
||||||
# without digits, we filter out many common branch names like
|
# between branches and tags. By ignoring refnames without digits, we
|
||||||
# "release" and "stabilization", as well as "HEAD" and "master".
|
# filter out many common branch names like "release" and
|
||||||
|
# "stabilization", as well as "HEAD" and "master".
|
||||||
|
tags = set([r for r in refs if re.search(r'\d', r)])
|
||||||
|
if verbose:
|
||||||
|
print("discarding '%s', no digits" % ",".join(refs - tags))
|
||||||
if verbose:
|
if verbose:
|
||||||
print("remaining refs: %s" % ",".join(sorted(refs)))
|
print("likely tags: %s" % ",".join(sorted(tags)))
|
||||||
for ref in sorted(refs):
|
for ref in sorted(tags):
|
||||||
# sorting will prefer e.g. "2.0" over "2.0rc1"
|
# sorting will prefer e.g. "2.0" over "2.0rc1"
|
||||||
if ref.startswith(tag_prefix):
|
if ref.startswith(tag_prefix):
|
||||||
r = ref[len(tag_prefix):]
|
r = ref[len(tag_prefix):]
|
||||||
if verbose:
|
if verbose:
|
||||||
print("picking %s" % r)
|
print("picking %s" % r)
|
||||||
return { "version": r,
|
return {"version": r,
|
||||||
"full": variables["full"].strip() }
|
"full-revisionid": keywords["full"].strip(),
|
||||||
# no suitable tags, so we use the full revision id
|
"dirty": False, "error": None,
|
||||||
|
"date": date}
|
||||||
|
# no suitable tags, so version is "0+unknown", but full hex is still there
|
||||||
if verbose:
|
if verbose:
|
||||||
print("no suitable tags, using full revision id")
|
print("no suitable tags, using unknown + full revision id")
|
||||||
return { "version": variables["full"].strip(),
|
return {"version": "0+unknown",
|
||||||
"full": variables["full"].strip() }
|
"full-revisionid": keywords["full"].strip(),
|
||||||
|
"dirty": False, "error": "no suitable tags", "date": None}
|
||||||
|
|
||||||
def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
|
|
||||||
# this runs 'git' from the root of the source tree. That either means
|
@register_vcs_handler("git", "pieces_from_vcs")
|
||||||
# someone ran a setup.py command (and this code is in versioneer.py, so
|
def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
|
||||||
# IN_LONG_VERSION_PY=False, thus the containing directory is the root of
|
"""Get version from 'git describe' in the root of the source tree.
|
||||||
# the source tree), or someone ran a project-specific entry point (and
|
|
||||||
# this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
|
This only gets called if the git-archive 'subst' keywords were *not*
|
||||||
# containing directory is somewhere deeper in the source tree). This only
|
expanded, and _version.py hasn't already been rewritten with a short
|
||||||
# gets called if the git-archive 'subst' variables were *not* expanded,
|
version string, meaning we're inside a checked out source tree.
|
||||||
# and _version.py hasn't already been rewritten with a short version
|
"""
|
||||||
# string, meaning we're inside a checked out source tree.
|
GITS = ["git"]
|
||||||
|
if sys.platform == "win32":
|
||||||
|
GITS = ["git.cmd", "git.exe"]
|
||||||
|
|
||||||
|
out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
|
||||||
|
hide_stderr=True)
|
||||||
|
if rc != 0:
|
||||||
|
if verbose:
|
||||||
|
print("Directory %s not under git control" % root)
|
||||||
|
raise NotThisMethod("'git rev-parse --git-dir' returned error")
|
||||||
|
|
||||||
|
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
|
||||||
|
# if there isn't one, this yields HEX[-dirty] (no NUM)
|
||||||
|
describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
|
||||||
|
"--always", "--long",
|
||||||
|
"--match", "%s*" % tag_prefix],
|
||||||
|
cwd=root)
|
||||||
|
# --long was added in git-1.5.5
|
||||||
|
if describe_out is None:
|
||||||
|
raise NotThisMethod("'git describe' failed")
|
||||||
|
describe_out = describe_out.strip()
|
||||||
|
full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
|
||||||
|
if full_out is None:
|
||||||
|
raise NotThisMethod("'git rev-parse' failed")
|
||||||
|
full_out = full_out.strip()
|
||||||
|
|
||||||
|
pieces = {}
|
||||||
|
pieces["long"] = full_out
|
||||||
|
pieces["short"] = full_out[:7] # maybe improved later
|
||||||
|
pieces["error"] = None
|
||||||
|
|
||||||
|
# parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
|
||||||
|
# TAG might have hyphens.
|
||||||
|
git_describe = describe_out
|
||||||
|
|
||||||
|
# look for -dirty suffix
|
||||||
|
dirty = git_describe.endswith("-dirty")
|
||||||
|
pieces["dirty"] = dirty
|
||||||
|
if dirty:
|
||||||
|
git_describe = git_describe[:git_describe.rindex("-dirty")]
|
||||||
|
|
||||||
|
# now we have TAG-NUM-gHEX or HEX
|
||||||
|
|
||||||
|
if "-" in git_describe:
|
||||||
|
# TAG-NUM-gHEX
|
||||||
|
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
|
||||||
|
if not mo:
|
||||||
|
# unparseable. Maybe git-describe is misbehaving?
|
||||||
|
pieces["error"] = ("unable to parse git-describe output: '%s'"
|
||||||
|
% describe_out)
|
||||||
|
return pieces
|
||||||
|
|
||||||
|
# tag
|
||||||
|
full_tag = mo.group(1)
|
||||||
|
if not full_tag.startswith(tag_prefix):
|
||||||
|
if verbose:
|
||||||
|
fmt = "tag '%s' doesn't start with prefix '%s'"
|
||||||
|
print(fmt % (full_tag, tag_prefix))
|
||||||
|
pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
|
||||||
|
% (full_tag, tag_prefix))
|
||||||
|
return pieces
|
||||||
|
pieces["closest-tag"] = full_tag[len(tag_prefix):]
|
||||||
|
|
||||||
|
# distance: number of commits since tag
|
||||||
|
pieces["distance"] = int(mo.group(2))
|
||||||
|
|
||||||
|
# commit: short hex revision ID
|
||||||
|
pieces["short"] = mo.group(3)
|
||||||
|
|
||||||
|
else:
|
||||||
|
# HEX: no tags
|
||||||
|
pieces["closest-tag"] = None
|
||||||
|
count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
|
||||||
|
cwd=root)
|
||||||
|
pieces["distance"] = int(count_out) # total number of commits
|
||||||
|
|
||||||
|
# commit date: see ISO-8601 comment in git_versions_from_keywords()
|
||||||
|
date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
|
||||||
|
cwd=root)[0].strip()
|
||||||
|
pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
|
||||||
|
|
||||||
|
return pieces
|
||||||
|
|
||||||
|
|
||||||
|
def plus_or_dot(pieces):
|
||||||
|
"""Return a + if we don't already have one, else return a ."""
|
||||||
|
if "+" in pieces.get("closest-tag", ""):
|
||||||
|
return "."
|
||||||
|
return "+"
|
||||||
|
|
||||||
|
|
||||||
|
def render_pep440(pieces):
|
||||||
|
"""Build up version string, with post-release "local version identifier".
|
||||||
|
|
||||||
|
Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
|
||||||
|
get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
|
||||||
|
|
||||||
|
Exceptions:
|
||||||
|
1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
|
||||||
|
"""
|
||||||
|
if pieces["closest-tag"]:
|
||||||
|
rendered = pieces["closest-tag"]
|
||||||
|
if pieces["distance"] or pieces["dirty"]:
|
||||||
|
rendered += plus_or_dot(pieces)
|
||||||
|
rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += ".dirty"
|
||||||
|
else:
|
||||||
|
# exception #1
|
||||||
|
rendered = "0+untagged.%d.g%s" % (pieces["distance"],
|
||||||
|
pieces["short"])
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += ".dirty"
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def render_pep440_pre(pieces):
|
||||||
|
"""TAG[.post.devDISTANCE] -- No -dirty.
|
||||||
|
|
||||||
|
Exceptions:
|
||||||
|
1: no tags. 0.post.devDISTANCE
|
||||||
|
"""
|
||||||
|
if pieces["closest-tag"]:
|
||||||
|
rendered = pieces["closest-tag"]
|
||||||
|
if pieces["distance"]:
|
||||||
|
rendered += ".post.dev%d" % pieces["distance"]
|
||||||
|
else:
|
||||||
|
# exception #1
|
||||||
|
rendered = "0.post.dev%d" % pieces["distance"]
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def render_pep440_post(pieces):
|
||||||
|
"""TAG[.postDISTANCE[.dev0]+gHEX] .
|
||||||
|
|
||||||
|
The ".dev0" means dirty. Note that .dev0 sorts backwards
|
||||||
|
(a dirty tree will appear "older" than the corresponding clean one),
|
||||||
|
but you shouldn't be releasing software with -dirty anyways.
|
||||||
|
|
||||||
|
Exceptions:
|
||||||
|
1: no tags. 0.postDISTANCE[.dev0]
|
||||||
|
"""
|
||||||
|
if pieces["closest-tag"]:
|
||||||
|
rendered = pieces["closest-tag"]
|
||||||
|
if pieces["distance"] or pieces["dirty"]:
|
||||||
|
rendered += ".post%d" % pieces["distance"]
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += ".dev0"
|
||||||
|
rendered += plus_or_dot(pieces)
|
||||||
|
rendered += "g%s" % pieces["short"]
|
||||||
|
else:
|
||||||
|
# exception #1
|
||||||
|
rendered = "0.post%d" % pieces["distance"]
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += ".dev0"
|
||||||
|
rendered += "+g%s" % pieces["short"]
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def render_pep440_old(pieces):
|
||||||
|
"""TAG[.postDISTANCE[.dev0]] .
|
||||||
|
|
||||||
|
The ".dev0" means dirty.
|
||||||
|
|
||||||
|
Eexceptions:
|
||||||
|
1: no tags. 0.postDISTANCE[.dev0]
|
||||||
|
"""
|
||||||
|
if pieces["closest-tag"]:
|
||||||
|
rendered = pieces["closest-tag"]
|
||||||
|
if pieces["distance"] or pieces["dirty"]:
|
||||||
|
rendered += ".post%d" % pieces["distance"]
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += ".dev0"
|
||||||
|
else:
|
||||||
|
# exception #1
|
||||||
|
rendered = "0.post%d" % pieces["distance"]
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += ".dev0"
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def render_git_describe(pieces):
|
||||||
|
"""TAG[-DISTANCE-gHEX][-dirty].
|
||||||
|
|
||||||
|
Like 'git describe --tags --dirty --always'.
|
||||||
|
|
||||||
|
Exceptions:
|
||||||
|
1: no tags. HEX[-dirty] (note: no 'g' prefix)
|
||||||
|
"""
|
||||||
|
if pieces["closest-tag"]:
|
||||||
|
rendered = pieces["closest-tag"]
|
||||||
|
if pieces["distance"]:
|
||||||
|
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
|
||||||
|
else:
|
||||||
|
# exception #1
|
||||||
|
rendered = pieces["short"]
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += "-dirty"
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def render_git_describe_long(pieces):
|
||||||
|
"""TAG-DISTANCE-gHEX[-dirty].
|
||||||
|
|
||||||
|
Like 'git describe --tags --dirty --always -long'.
|
||||||
|
The distance/hash is unconditional.
|
||||||
|
|
||||||
|
Exceptions:
|
||||||
|
1: no tags. HEX[-dirty] (note: no 'g' prefix)
|
||||||
|
"""
|
||||||
|
if pieces["closest-tag"]:
|
||||||
|
rendered = pieces["closest-tag"]
|
||||||
|
rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
|
||||||
|
else:
|
||||||
|
# exception #1
|
||||||
|
rendered = pieces["short"]
|
||||||
|
if pieces["dirty"]:
|
||||||
|
rendered += "-dirty"
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def render(pieces, style):
|
||||||
|
"""Render the given version pieces into the requested style."""
|
||||||
|
if pieces["error"]:
|
||||||
|
return {"version": "unknown",
|
||||||
|
"full-revisionid": pieces.get("long"),
|
||||||
|
"dirty": None,
|
||||||
|
"error": pieces["error"],
|
||||||
|
"date": None}
|
||||||
|
|
||||||
|
if not style or style == "default":
|
||||||
|
style = "pep440" # the default
|
||||||
|
|
||||||
|
if style == "pep440":
|
||||||
|
rendered = render_pep440(pieces)
|
||||||
|
elif style == "pep440-pre":
|
||||||
|
rendered = render_pep440_pre(pieces)
|
||||||
|
elif style == "pep440-post":
|
||||||
|
rendered = render_pep440_post(pieces)
|
||||||
|
elif style == "pep440-old":
|
||||||
|
rendered = render_pep440_old(pieces)
|
||||||
|
elif style == "git-describe":
|
||||||
|
rendered = render_git_describe(pieces)
|
||||||
|
elif style == "git-describe-long":
|
||||||
|
rendered = render_git_describe_long(pieces)
|
||||||
|
else:
|
||||||
|
raise ValueError("unknown style '%s'" % style)
|
||||||
|
|
||||||
|
return {"version": rendered, "full-revisionid": pieces["long"],
|
||||||
|
"dirty": pieces["dirty"], "error": None,
|
||||||
|
"date": pieces.get("date")}
|
||||||
|
|
||||||
|
|
||||||
|
def get_versions():
|
||||||
|
"""Get version information or return default if unable to do so."""
|
||||||
|
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
|
||||||
|
# __file__, we can work backwards from there to the root. Some
|
||||||
|
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
|
||||||
|
# case we can only use expanded keywords.
|
||||||
|
|
||||||
|
cfg = get_config()
|
||||||
|
verbose = cfg.verbose
|
||||||
|
|
||||||
try:
|
try:
|
||||||
here = os.path.abspath(__file__)
|
return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
|
||||||
except NameError:
|
verbose)
|
||||||
# some py2exe/bbfreeze/non-CPython implementations don't do __file__
|
except NotThisMethod:
|
||||||
return {} # not always correct
|
pass
|
||||||
|
|
||||||
# versionfile_source is the relative path from the top of the source tree
|
try:
|
||||||
# (where the .git directory might live) to this file. Invert this to find
|
root = os.path.realpath(__file__)
|
||||||
# the root from __file__.
|
|
||||||
root = here
|
|
||||||
if IN_LONG_VERSION_PY:
|
|
||||||
for i in range(len(versionfile_source.split("/"))):
|
|
||||||
root = os.path.dirname(root)
|
|
||||||
else:
|
|
||||||
root = os.path.dirname(here)
|
|
||||||
if not os.path.exists(os.path.join(root, ".git")):
|
|
||||||
if verbose:
|
|
||||||
print("no .git in %s" % root)
|
|
||||||
return {}
|
|
||||||
|
|
||||||
GIT = "git"
|
|
||||||
if sys.platform == "win32":
|
|
||||||
GIT = "git.cmd"
|
|
||||||
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
|
|
||||||
cwd=root)
|
|
||||||
if stdout is None:
|
|
||||||
return {}
|
|
||||||
if not stdout.startswith(tag_prefix):
|
|
||||||
if verbose:
|
|
||||||
print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
|
|
||||||
return {}
|
|
||||||
tag = stdout[len(tag_prefix):]
|
|
||||||
stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
|
|
||||||
if stdout is None:
|
|
||||||
return {}
|
|
||||||
full = stdout.strip()
|
|
||||||
if tag.endswith("-dirty"):
|
|
||||||
full += "-dirty"
|
|
||||||
return {"version": tag, "full": full}
|
|
||||||
|
|
||||||
|
|
||||||
def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
|
|
||||||
if IN_LONG_VERSION_PY:
|
|
||||||
# We're running from _version.py. If it's from a source tree
|
|
||||||
# (execute-in-place), we can work upwards to find the root of the
|
|
||||||
# tree, and then check the parent directory for a version string. If
|
|
||||||
# it's in an installed application, there's no hope.
|
|
||||||
try:
|
|
||||||
here = os.path.abspath(__file__)
|
|
||||||
except NameError:
|
|
||||||
# py2exe/bbfreeze/non-CPython don't have __file__
|
|
||||||
return {} # without __file__, we have no hope
|
|
||||||
# versionfile_source is the relative path from the top of the source
|
# versionfile_source is the relative path from the top of the source
|
||||||
# tree to _version.py. Invert this to find the root from __file__.
|
# tree (where the .git directory might live) to this file. Invert
|
||||||
root = here
|
# this to find the root from __file__.
|
||||||
for i in range(len(versionfile_source.split("/"))):
|
for i in cfg.versionfile_source.split('/'):
|
||||||
root = os.path.dirname(root)
|
root = os.path.dirname(root)
|
||||||
else:
|
except NameError:
|
||||||
# we're running from versioneer.py, which means we're running from
|
return {"version": "0+unknown", "full-revisionid": None,
|
||||||
# the setup.py in a source tree. sys.argv[0] is setup.py in the root.
|
"dirty": None,
|
||||||
here = os.path.abspath(sys.argv[0])
|
"error": "unable to find root of source tree",
|
||||||
root = os.path.dirname(here)
|
"date": None}
|
||||||
|
|
||||||
# Source tarballs conventionally unpack into a directory that includes
|
try:
|
||||||
# both the project name and a version string.
|
pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
|
||||||
dirname = os.path.basename(root)
|
return render(pieces, cfg.style)
|
||||||
if not dirname.startswith(parentdir_prefix):
|
except NotThisMethod:
|
||||||
if verbose:
|
pass
|
||||||
print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
|
|
||||||
(root, dirname, parentdir_prefix))
|
|
||||||
return None
|
|
||||||
return {"version": dirname[len(parentdir_prefix):], "full": ""}
|
|
||||||
|
|
||||||
tag_prefix = "nilmrun-"
|
try:
|
||||||
parentdir_prefix = "nilmrun-"
|
if cfg.parentdir_prefix:
|
||||||
versionfile_source = "nilmrun/_version.py"
|
return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
|
||||||
|
except NotThisMethod:
|
||||||
def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
|
pass
|
||||||
variables = { "refnames": git_refnames, "full": git_full }
|
|
||||||
ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
|
|
||||||
if not ver:
|
|
||||||
ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
|
|
||||||
if not ver:
|
|
||||||
ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
|
|
||||||
verbose)
|
|
||||||
if not ver:
|
|
||||||
ver = default
|
|
||||||
return ver
|
|
||||||
|
|
||||||
|
return {"version": "0+unknown", "full-revisionid": None,
|
||||||
|
"dirty": None,
|
||||||
|
"error": "unable to compute version", "date": None}
|
||||||
|
|
|
@ -1,10 +1,8 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from nilmdb.utils.printf import *
|
|
||||||
|
|
||||||
import threading
|
import threading
|
||||||
import subprocess
|
import subprocess
|
||||||
import cStringIO
|
import io
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import signal
|
import signal
|
||||||
|
@ -15,16 +13,18 @@ import tempfile
|
||||||
import atexit
|
import atexit
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
|
|
||||||
class ProcessError(Exception):
|
class ProcessError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class LogReceiver(object):
|
class LogReceiver(object):
|
||||||
"""Spawn a thread that listens to a pipe for log messages,
|
"""Spawn a thread that listens to a pipe for log messages,
|
||||||
and stores them locally."""
|
and stores them locally."""
|
||||||
def __init__(self, pipe):
|
def __init__(self, pipe):
|
||||||
self.pipe = pipe
|
self.pipe = pipe
|
||||||
self.log = cStringIO.StringIO()
|
self.log = io.BytesIO()
|
||||||
self.thread = threading.Thread(target = self.run)
|
self.thread = threading.Thread(target=self.run)
|
||||||
self.thread.start()
|
self.thread.start()
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
@ -39,11 +39,12 @@ class LogReceiver(object):
|
||||||
return self.log.getvalue()
|
return self.log.getvalue()
|
||||||
|
|
||||||
def clear(self):
|
def clear(self):
|
||||||
self.log = cStringIO.StringIO()
|
self.log = io.BytesIO()
|
||||||
|
|
||||||
|
|
||||||
class Process(object):
|
class Process(object):
|
||||||
"""Spawn and manage a subprocess, and capture its output."""
|
"""Spawn and manage a subprocess, and capture its output."""
|
||||||
def __init__(self, argv, tempfile = None):
|
def __init__(self, argv, tempfile=None):
|
||||||
self.start_time = None
|
self.start_time = None
|
||||||
|
|
||||||
# Use a pipe for communicating log data
|
# Use a pipe for communicating log data
|
||||||
|
@ -55,9 +56,9 @@ class Process(object):
|
||||||
|
|
||||||
# Spawn the new process
|
# Spawn the new process
|
||||||
try:
|
try:
|
||||||
self._process = subprocess.Popen(args = argv, stdin = nullfd,
|
self._process = subprocess.Popen(args=argv, stdin=nullfd,
|
||||||
stdout = wpipe, stderr = wpipe,
|
stdout=wpipe, stderr=wpipe,
|
||||||
close_fds = True, cwd = "/tmp")
|
close_fds=True, cwd="/tmp")
|
||||||
except (OSError, TypeError) as e:
|
except (OSError, TypeError) as e:
|
||||||
raise ProcessError(str(e))
|
raise ProcessError(str(e))
|
||||||
finally:
|
finally:
|
||||||
|
@ -69,7 +70,7 @@ class Process(object):
|
||||||
self.start_time = time.time()
|
self.start_time = time.time()
|
||||||
self.pid = str(uuid.uuid1(self._process.pid or 0))
|
self.pid = str(uuid.uuid1(self._process.pid or 0))
|
||||||
|
|
||||||
def _join(self, timeout = 1.0):
|
def _join(self, timeout=1.0):
|
||||||
start = time.time()
|
start = time.time()
|
||||||
while True:
|
while True:
|
||||||
if self._process.poll() is not None:
|
if self._process.poll() is not None:
|
||||||
|
@ -78,7 +79,7 @@ class Process(object):
|
||||||
return False
|
return False
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
|
|
||||||
def terminate(self, timeout = 1.0):
|
def terminate(self, timeout=1.0):
|
||||||
"""Terminate a process, and all of its children that are in the same
|
"""Terminate a process, and all of its children that are in the same
|
||||||
process group."""
|
process group."""
|
||||||
try:
|
try:
|
||||||
|
@ -89,19 +90,19 @@ class Process(object):
|
||||||
def getpgid(pid):
|
def getpgid(pid):
|
||||||
try:
|
try:
|
||||||
return os.getpgid(pid)
|
return os.getpgid(pid)
|
||||||
except OSError: # pragma: no cover
|
except OSError: # pragma: no cover
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def kill(pid, sig):
|
def kill(pid, sig):
|
||||||
try:
|
try:
|
||||||
return os.kill(pid, sig)
|
return os.kill(pid, sig)
|
||||||
except OSError: # pragma: no cover
|
except OSError: # pragma: no cover
|
||||||
return
|
return
|
||||||
|
|
||||||
# Find all children
|
# Find all children
|
||||||
group = getpgid(self._process.pid)
|
group = getpgid(self._process.pid)
|
||||||
main = psutil.Process(self._process.pid)
|
main = psutil.Process(self._process.pid)
|
||||||
allproc = [ main ] + main.get_children(recursive = True)
|
allproc = [main] + main.children(recursive=True)
|
||||||
|
|
||||||
# Kill with SIGTERM, if they're still in this process group
|
# Kill with SIGTERM, if they're still in this process group
|
||||||
for proc in allproc:
|
for proc in allproc:
|
||||||
|
@ -119,7 +120,7 @@ class Process(object):
|
||||||
|
|
||||||
# See if it worked
|
# See if it worked
|
||||||
return self._join(timeout)
|
return self._join(timeout)
|
||||||
except psutil.Error: # pragma: no cover (race condition)
|
except psutil.Error: # pragma: no cover (race condition)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def clear_log(self):
|
def clear_log(self):
|
||||||
|
@ -142,22 +143,22 @@ class Process(object):
|
||||||
Call .get_info() about a second later."""
|
Call .get_info() about a second later."""
|
||||||
try:
|
try:
|
||||||
main = psutil.Process(self._process.pid)
|
main = psutil.Process(self._process.pid)
|
||||||
self._process_list = [ main ] + main.get_children(recursive = True)
|
self._process_list = [main] + main.children(recursive=True)
|
||||||
for proc in self._process_list:
|
for proc in self._process_list:
|
||||||
proc.get_cpu_percent(0)
|
proc.cpu_percent(0)
|
||||||
except psutil.Error: # pragma: no cover (race condition)
|
except psutil.Error: # pragma: no cover (race condition)
|
||||||
self._process_list = [ ]
|
self._process_list = []
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_empty_info():
|
def get_empty_info():
|
||||||
return { "cpu_percent": 0,
|
return {"cpu_percent": 0,
|
||||||
"cpu_user": 0,
|
"cpu_user": 0,
|
||||||
"cpu_sys": 0,
|
"cpu_sys": 0,
|
||||||
"mem_phys": 0,
|
"mem_phys": 0,
|
||||||
"mem_virt": 0,
|
"mem_virt": 0,
|
||||||
"io_read": 0,
|
"io_read": 0,
|
||||||
"io_write": 0,
|
"io_write": 0,
|
||||||
"procs": 0 }
|
"procs": 0}
|
||||||
|
|
||||||
def get_info(self):
|
def get_info(self):
|
||||||
"""Return a dictionary with info about the process CPU and memory
|
"""Return a dictionary with info about the process CPU and memory
|
||||||
|
@ -165,14 +166,14 @@ class Process(object):
|
||||||
d = self.get_empty_info()
|
d = self.get_empty_info()
|
||||||
for proc in self._process_list:
|
for proc in self._process_list:
|
||||||
try:
|
try:
|
||||||
d["cpu_percent"] += proc.get_cpu_percent(0)
|
d["cpu_percent"] += proc.cpu_percent(0)
|
||||||
cpuinfo = proc.get_cpu_times()
|
cpuinfo = proc.cpu_times()
|
||||||
d["cpu_user"] += cpuinfo.user
|
d["cpu_user"] += cpuinfo.user
|
||||||
d["cpu_sys"] += cpuinfo.system
|
d["cpu_sys"] += cpuinfo.system
|
||||||
meminfo = proc.get_memory_info()
|
meminfo = proc.memory_info()
|
||||||
d["mem_phys"] += meminfo.rss
|
d["mem_phys"] += meminfo.rss
|
||||||
d["mem_virt"] += meminfo.vms
|
d["mem_virt"] += meminfo.vms
|
||||||
ioinfo = proc.get_io_counters()
|
ioinfo = proc.io_counters()
|
||||||
d["io_read"] += ioinfo.read_bytes
|
d["io_read"] += ioinfo.read_bytes
|
||||||
d["io_write"] += ioinfo.write_bytes
|
d["io_write"] += ioinfo.write_bytes
|
||||||
d["procs"] += 1
|
d["procs"] += 1
|
||||||
|
@ -180,6 +181,7 @@ class Process(object):
|
||||||
pass
|
pass
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
class ProcessManager(object):
|
class ProcessManager(object):
|
||||||
"""Track and manage a collection of Process objects"""
|
"""Track and manage a collection of Process objects"""
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
@ -191,23 +193,23 @@ class ProcessManager(object):
|
||||||
if pid in self.tmpdirs:
|
if pid in self.tmpdirs:
|
||||||
try:
|
try:
|
||||||
shutil.rmtree(self.tmpdirs[pid])
|
shutil.rmtree(self.tmpdirs[pid])
|
||||||
except OSError: # pragma: no cover
|
except OSError: # pragma: no cover
|
||||||
pass
|
pass
|
||||||
del self.tmpdirs[pid]
|
del self.tmpdirs[pid]
|
||||||
|
|
||||||
def _atexit(self):
|
def _atexit(self):
|
||||||
# Kill remaining processes, remove their dirs
|
# Kill remaining processes, remove their dirs
|
||||||
for pid in self.processes.keys():
|
for pid in list(self.processes.keys()):
|
||||||
try:
|
try:
|
||||||
self.processes[pid].terminate()
|
self.processes[pid].terminate()
|
||||||
del self.processes[pid]
|
del self.processes[pid]
|
||||||
shutil.rmtree(self.tmpdirs[pid])
|
shutil.rmtree(self.tmpdirs[pid])
|
||||||
del self.tmpdirs[pid]
|
del self.tmpdirs[pid]
|
||||||
except Exception: # pragma: no cover
|
except Exception: # pragma: no cover
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
return iter(self.processes.keys())
|
return iter(list(self.processes.keys()))
|
||||||
|
|
||||||
def __getitem__(self, key):
|
def __getitem__(self, key):
|
||||||
return self.processes[key]
|
return self.processes[key]
|
||||||
|
@ -218,7 +220,7 @@ class ProcessManager(object):
|
||||||
accessible in the code as sys.argv[1:]."""
|
accessible in the code as sys.argv[1:]."""
|
||||||
# The easiest way to do this, by far, is to just write the
|
# The easiest way to do this, by far, is to just write the
|
||||||
# code to a file. Make a directory to put it in.
|
# code to a file. Make a directory to put it in.
|
||||||
tmpdir = tempfile.mkdtemp(prefix = "nilmrun-usercode-")
|
tmpdir = tempfile.mkdtemp(prefix="nilmrun-usercode-")
|
||||||
try:
|
try:
|
||||||
# Write the code
|
# Write the code
|
||||||
codepath = os.path.join(tmpdir, "usercode.py")
|
codepath = os.path.join(tmpdir, "usercode.py")
|
||||||
|
@ -229,7 +231,7 @@ class ProcessManager(object):
|
||||||
f.write(repr(args))
|
f.write(repr(args))
|
||||||
|
|
||||||
# Run the code
|
# Run the code
|
||||||
argv = [ sys.executable, "-B", "-s", "-u", codepath ] + args
|
argv = [sys.executable, "-B", "-s", "-u", codepath] + args
|
||||||
pid = self.run_command(argv)
|
pid = self.run_command(argv)
|
||||||
|
|
||||||
# Save the temp dir
|
# Save the temp dir
|
||||||
|
@ -242,7 +244,7 @@ class ProcessManager(object):
|
||||||
if tmpdir is not None:
|
if tmpdir is not None:
|
||||||
try:
|
try:
|
||||||
shutil.rmtree(tmpdir)
|
shutil.rmtree(tmpdir)
|
||||||
except OSError: # pragma: no cover
|
except OSError: # pragma: no cover
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def run_command(self, argv):
|
def run_command(self, argv):
|
||||||
|
@ -260,28 +262,27 @@ class ProcessManager(object):
|
||||||
|
|
||||||
def get_info(self):
|
def get_info(self):
|
||||||
"""Get info about all running PIDs"""
|
"""Get info about all running PIDs"""
|
||||||
info = { "total" : Process.get_empty_info(),
|
info = {
|
||||||
"pids" : {},
|
"total": Process.get_empty_info(),
|
||||||
"system" : {}
|
"pids": {},
|
||||||
}
|
"system": {}
|
||||||
|
}
|
||||||
|
|
||||||
# Trigger CPU usage collection
|
# Trigger CPU usage collection
|
||||||
for pid in self:
|
for pid in self:
|
||||||
self[pid].get_info_prepare()
|
self[pid].get_info_prepare()
|
||||||
psutil.cpu_percent(0, percpu = True)
|
psutil.cpu_percent(0, percpu=True)
|
||||||
|
|
||||||
# Give it some time
|
# Give it some time
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
# Retrieve info for system
|
# Retrieve info for system
|
||||||
info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True))
|
info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True))
|
||||||
info["system"]["cpu_max"] = 100.0 * psutil.NUM_CPUS
|
info["system"]["cpu_max"] = 100.0 * psutil.cpu_count()
|
||||||
info["system"]["procs"] = len(psutil.get_pid_list())
|
info["system"]["procs"] = len(psutil.pids())
|
||||||
# psutil > 0.6.0's psutil.virtual_memory() would be better here,
|
meminfo = psutil.virtual_memory()
|
||||||
# but this should give the same info.
|
|
||||||
meminfo = psutil.phymem_usage()
|
|
||||||
info["system"]["mem_total"] = meminfo.total
|
info["system"]["mem_total"] = meminfo.total
|
||||||
info["system"]["mem_used"] = int(meminfo.total * meminfo.percent / 100)
|
info["system"]["mem_used"] = meminfo.used
|
||||||
|
|
||||||
# Retrieve info for each PID
|
# Retrieve info for each PID
|
||||||
for pid in self:
|
for pid in self:
|
||||||
|
|
|
@ -1,19 +1,12 @@
|
||||||
"""CherryPy-based server for running NILM filters via HTTP"""
|
"""CherryPy-based server for running NILM filters via HTTP"""
|
||||||
|
|
||||||
import cherrypy
|
import cherrypy
|
||||||
import sys
|
|
||||||
import os
|
import os
|
||||||
import socket
|
import socket
|
||||||
import simplejson as json
|
|
||||||
import traceback
|
import traceback
|
||||||
import time
|
|
||||||
|
|
||||||
import nilmdb
|
from nilmdb.utils.printf import sprintf
|
||||||
from nilmdb.utils.printf import *
|
|
||||||
from nilmdb.server.serverutil import (
|
from nilmdb.server.serverutil import (
|
||||||
chunked_response,
|
|
||||||
response_type,
|
|
||||||
workaround_cp_bug_1200,
|
|
||||||
exception_to_httperror,
|
exception_to_httperror,
|
||||||
CORS_allow,
|
CORS_allow,
|
||||||
json_to_request_params,
|
json_to_request_params,
|
||||||
|
@ -24,11 +17,12 @@ from nilmdb.server.serverutil import (
|
||||||
)
|
)
|
||||||
from nilmdb.utils import serializer_proxy
|
from nilmdb.utils import serializer_proxy
|
||||||
import nilmrun
|
import nilmrun
|
||||||
import nilmrun.testfilter
|
import nilmrun.processmanager
|
||||||
|
|
||||||
# Add CORS_allow tool
|
# Add CORS_allow tool
|
||||||
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
|
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
|
||||||
|
|
||||||
|
|
||||||
# CherryPy apps
|
# CherryPy apps
|
||||||
class App(object):
|
class App(object):
|
||||||
"""Root application for NILM runner"""
|
"""Root application for NILM runner"""
|
||||||
|
@ -55,6 +49,7 @@ class App(object):
|
||||||
def version(self):
|
def version(self):
|
||||||
return nilmrun.__version__
|
return nilmrun.__version__
|
||||||
|
|
||||||
|
|
||||||
class AppProcess(object):
|
class AppProcess(object):
|
||||||
|
|
||||||
def __init__(self, manager):
|
def __init__(self, manager):
|
||||||
|
@ -76,7 +71,7 @@ class AppProcess(object):
|
||||||
# /process/status
|
# /process/status
|
||||||
@cherrypy.expose
|
@cherrypy.expose
|
||||||
@cherrypy.tools.json_out()
|
@cherrypy.tools.json_out()
|
||||||
def status(self, pid, clear = False):
|
def status(self, pid, clear=False):
|
||||||
"""Return status about a process. If clear = True, also clear
|
"""Return status about a process. If clear = True, also clear
|
||||||
the log."""
|
the log."""
|
||||||
clear = bool_param(clear)
|
clear = bool_param(clear)
|
||||||
|
@ -106,18 +101,19 @@ class AppProcess(object):
|
||||||
@cherrypy.expose
|
@cherrypy.expose
|
||||||
@cherrypy.tools.json_in()
|
@cherrypy.tools.json_in()
|
||||||
@cherrypy.tools.json_out()
|
@cherrypy.tools.json_out()
|
||||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||||
def remove(self, pid):
|
def remove(self, pid):
|
||||||
"""Remove a process from the manager, killing it if necessary."""
|
"""Remove a process from the manager, killing it if necessary."""
|
||||||
if pid not in self.manager:
|
if pid not in self.manager:
|
||||||
raise cherrypy.HTTPError("404 Not Found", "No such PID")
|
raise cherrypy.HTTPError("404 Not Found", "No such PID")
|
||||||
if not self.manager.terminate(pid): # pragma: no cover
|
if not self.manager.terminate(pid): # pragma: no cover
|
||||||
raise cherrypy.HTTPError("503 Service Unavailable",
|
raise cherrypy.HTTPError("503 Service Unavailable",
|
||||||
"Failed to stop process")
|
"Failed to stop process")
|
||||||
status = self.process_status(pid)
|
status = self.process_status(pid)
|
||||||
self.manager.remove(pid)
|
self.manager.remove(pid)
|
||||||
return status
|
return status
|
||||||
|
|
||||||
|
|
||||||
class AppRun(object):
|
class AppRun(object):
|
||||||
def __init__(self, manager):
|
def __init__(self, manager):
|
||||||
self.manager = manager
|
self.manager = manager
|
||||||
|
@ -127,7 +123,7 @@ class AppRun(object):
|
||||||
@cherrypy.tools.json_in()
|
@cherrypy.tools.json_in()
|
||||||
@cherrypy.tools.json_out()
|
@cherrypy.tools.json_out()
|
||||||
@exception_to_httperror(nilmrun.processmanager.ProcessError)
|
@exception_to_httperror(nilmrun.processmanager.ProcessError)
|
||||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||||
def command(self, argv):
|
def command(self, argv):
|
||||||
"""Execute an arbitrary program on the server. argv is a
|
"""Execute an arbitrary program on the server. argv is a
|
||||||
list of the program and its arguments: 'argv[0]' is the program
|
list of the program and its arguments: 'argv[0]' is the program
|
||||||
|
@ -142,8 +138,8 @@ class AppRun(object):
|
||||||
@cherrypy.tools.json_in()
|
@cherrypy.tools.json_in()
|
||||||
@cherrypy.tools.json_out()
|
@cherrypy.tools.json_out()
|
||||||
@exception_to_httperror(nilmrun.processmanager.ProcessError)
|
@exception_to_httperror(nilmrun.processmanager.ProcessError)
|
||||||
@cherrypy.tools.CORS_allow(methods = ["POST"])
|
@cherrypy.tools.CORS_allow(methods=["POST"])
|
||||||
def code(self, code, args = None):
|
def code(self, code, args=None):
|
||||||
"""Execute arbitrary Python code. 'code' is a formatted string.
|
"""Execute arbitrary Python code. 'code' is a formatted string.
|
||||||
It will be run as if it were written into a Python file and
|
It will be run as if it were written into a Python file and
|
||||||
executed. 'args' is a list of strings, and they are passed
|
executed. 'args' is a list of strings, and they are passed
|
||||||
|
@ -156,23 +152,21 @@ class AppRun(object):
|
||||||
"args must be a list of strings")
|
"args must be a list of strings")
|
||||||
return self.manager.run_code(code, args)
|
return self.manager.run_code(code, args)
|
||||||
|
|
||||||
|
|
||||||
class Server(object):
|
class Server(object):
|
||||||
def __init__(self, host = '127.0.0.1', port = 8080,
|
def __init__(self, host='127.0.0.1', port=8080,
|
||||||
embedded = True, # hide diagnostics and output, etc
|
force_traceback=False, # include traceback in all errors
|
||||||
force_traceback = False, # include traceback in all errors
|
basepath='', # base URL path for cherrypy.tree
|
||||||
basepath = '', # base URL path for cherrypy.tree
|
|
||||||
):
|
):
|
||||||
self.embedded = embedded
|
|
||||||
|
|
||||||
# Build up global server configuration
|
# Build up global server configuration
|
||||||
cherrypy.config.update({
|
cherrypy.config.update({
|
||||||
|
'environment': 'embedded',
|
||||||
'server.socket_host': host,
|
'server.socket_host': host,
|
||||||
'server.socket_port': port,
|
'server.socket_port': port,
|
||||||
'engine.autoreload_on': False,
|
'engine.autoreload_on': False,
|
||||||
'server.max_request_body_size': 8*1024*1024,
|
'server.max_request_body_size': 8*1024*1024,
|
||||||
})
|
})
|
||||||
if self.embedded:
|
|
||||||
cherrypy.config.update({ 'environment': 'embedded' })
|
|
||||||
|
|
||||||
# Build up application specific configuration
|
# Build up application specific configuration
|
||||||
app_config = {}
|
app_config = {}
|
||||||
|
@ -181,23 +175,25 @@ class Server(object):
|
||||||
})
|
})
|
||||||
|
|
||||||
# Some default headers to just help identify that things are working
|
# Some default headers to just help identify that things are working
|
||||||
app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
|
app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
|
||||||
|
|
||||||
# Set up Cross-Origin Resource Sharing (CORS) handler so we
|
# Set up Cross-Origin Resource Sharing (CORS) handler so we
|
||||||
# can correctly respond to browsers' CORS preflight requests.
|
# can correctly respond to browsers' CORS preflight requests.
|
||||||
# This also limits verbs to GET and HEAD by default.
|
# This also limits verbs to GET and HEAD by default.
|
||||||
app_config.update({ 'tools.CORS_allow.on': True,
|
app_config.update({
|
||||||
'tools.CORS_allow.methods': ['GET', 'HEAD'] })
|
'tools.CORS_allow.on': True,
|
||||||
|
'tools.CORS_allow.methods': ['GET', 'HEAD']
|
||||||
|
})
|
||||||
|
|
||||||
# Configure the 'json_in' tool to also allow other content-types
|
# Configure the 'json_in' tool to also allow other content-types
|
||||||
# (like x-www-form-urlencoded), and to treat JSON as a dict that
|
# (like x-www-form-urlencoded), and to treat JSON as a dict that
|
||||||
# fills requests.param.
|
# fills requests.param.
|
||||||
app_config.update({ 'tools.json_in.force': False,
|
app_config.update({'tools.json_in.force': False,
|
||||||
'tools.json_in.processor': json_to_request_params })
|
'tools.json_in.processor': json_to_request_params})
|
||||||
|
|
||||||
# Send tracebacks in error responses. They're hidden by the
|
# Send tracebacks in error responses. They're hidden by the
|
||||||
# error_page function for client errors (code 400-499).
|
# error_page function for client errors (code 400-499).
|
||||||
app_config.update({ 'request.show_tracebacks' : True })
|
app_config.update({'request.show_tracebacks': True})
|
||||||
self.force_traceback = force_traceback
|
self.force_traceback = force_traceback
|
||||||
|
|
||||||
# Patch CherryPy error handler to never pad out error messages.
|
# Patch CherryPy error handler to never pad out error messages.
|
||||||
|
@ -215,7 +211,7 @@ class Server(object):
|
||||||
root.process = AppProcess(manager)
|
root.process = AppProcess(manager)
|
||||||
root.run = AppRun(manager)
|
root.run = AppRun(manager)
|
||||||
cherrypy.tree.apps = {}
|
cherrypy.tree.apps = {}
|
||||||
cherrypy.tree.mount(root, basepath, config = { "/" : app_config })
|
cherrypy.tree.mount(root, basepath, config={"/": app_config})
|
||||||
|
|
||||||
# Set up the WSGI application pointer for external programs
|
# Set up the WSGI application pointer for external programs
|
||||||
self.wsgi_application = cherrypy.tree
|
self.wsgi_application = cherrypy.tree
|
||||||
|
@ -225,17 +221,20 @@ class Server(object):
|
||||||
return json_error_page(status, message, traceback, version,
|
return json_error_page(status, message, traceback, version,
|
||||||
self.force_traceback)
|
self.force_traceback)
|
||||||
|
|
||||||
def start(self, blocking = False, event = None):
|
def start(self, blocking=False, event=None):
|
||||||
cherrypy_start(blocking, event, self.embedded)
|
cherrypy_start(blocking, event)
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
cherrypy_stop()
|
cherrypy_stop()
|
||||||
|
|
||||||
|
|
||||||
# Multiple processes and threads should be OK here, but we'll still
|
# Multiple processes and threads should be OK here, but we'll still
|
||||||
# follow the NilmDB approach of having just one globally initialized
|
# follow the NilmDB approach of having just one globally initialized
|
||||||
# copy of the server object.
|
# copy of the server object.
|
||||||
_wsgi_server = None
|
_wsgi_server = None
|
||||||
def wsgi_application(basepath): # pragma: no cover
|
|
||||||
|
|
||||||
|
def wsgi_application(basepath): # pragma: no cover
|
||||||
"""Return a WSGI application object.
|
"""Return a WSGI application object.
|
||||||
|
|
||||||
'basepath' is the URL path of the application base, which
|
'basepath' is the URL path of the application base, which
|
||||||
|
@ -248,13 +247,11 @@ def wsgi_application(basepath): # pragma: no cover
|
||||||
# Try to start the server
|
# Try to start the server
|
||||||
try:
|
try:
|
||||||
_wsgi_server = nilmrun.server.Server(
|
_wsgi_server = nilmrun.server.Server(
|
||||||
embedded = True,
|
basepath=basepath.rstrip('/'))
|
||||||
basepath = basepath.rstrip('/'))
|
|
||||||
except Exception:
|
except Exception:
|
||||||
# Build an error message on failure
|
# Build an error message on failure
|
||||||
import pprint
|
import pprint
|
||||||
err = sprintf("Initializing nilmrun failed:\n\n",
|
err = "Initializing nilmrun failed:\n\n"
|
||||||
dbpath)
|
|
||||||
err += traceback.format_exc()
|
err += traceback.format_exc()
|
||||||
try:
|
try:
|
||||||
import pwd
|
import pwd
|
||||||
|
@ -269,8 +266,10 @@ def wsgi_application(basepath): # pragma: no cover
|
||||||
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
|
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
|
||||||
if _wsgi_server is None:
|
if _wsgi_server is None:
|
||||||
# Serve up the error with our own mini WSGI app.
|
# Serve up the error with our own mini WSGI app.
|
||||||
headers = [ ('Content-type', 'text/plain'),
|
headers = [
|
||||||
('Content-length', str(len(err))) ]
|
('Content-type', 'text/plain'),
|
||||||
|
('Content-length', str(len(err)))
|
||||||
|
]
|
||||||
start_response("500 Internal Server Error", headers)
|
start_response("500 Internal Server Error", headers)
|
||||||
return [err]
|
return [err]
|
||||||
|
|
||||||
|
|
|
@ -1,22 +0,0 @@
|
||||||
#!/usr/bin/python
|
|
||||||
|
|
||||||
from nilmdb.utils.printf import *
|
|
||||||
import time
|
|
||||||
import signal
|
|
||||||
import sys
|
|
||||||
|
|
||||||
# This is just for testing the process management.
|
|
||||||
def test(n):
|
|
||||||
n = int(n)
|
|
||||||
if n < 0: # raise an exception
|
|
||||||
raise Exception("test exception")
|
|
||||||
if n == 0: # ignore SIGTERM and count to 100
|
|
||||||
n = 100
|
|
||||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
||||||
for x in range(n):
|
|
||||||
s = sprintf("dummy %d\n", x)
|
|
||||||
if x & 1:
|
|
||||||
sys.stdout.write(s)
|
|
||||||
else:
|
|
||||||
sys.stderr.write(s)
|
|
||||||
time.sleep(0.1)
|
|
2
requirements.txt
Normal file
2
requirements.txt
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
nilmdb>=2.0.3
|
||||||
|
psutil==5.7.2
|
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
|
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
|
||||||
from nilmdb.utils.printf import *
|
from nilmdb.utils.printf import *
|
||||||
|
@ -14,8 +14,8 @@ def main():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description = 'Kill/remove a process from the NilmRun server',
|
description = 'Kill/remove a process from the NilmRun server',
|
||||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
||||||
parser.add_argument("-V", "--version", action="version",
|
parser.add_argument("-v", "--version", action="version",
|
||||||
version = nilmrun.__version__)
|
version=nilmrun.__version__)
|
||||||
group = parser.add_argument_group("Standard options")
|
group = parser.add_argument_group("Standard options")
|
||||||
group.add_argument('-u', '--url',
|
group.add_argument('-u', '--url',
|
||||||
help = 'NilmRun server URL', default = def_url)
|
help = 'NilmRun server URL', default = def_url)
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import nilmrun.server
|
import nilmrun.server
|
||||||
import argparse
|
import argparse
|
||||||
|
@ -11,9 +11,8 @@ def main():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description = 'Run the NilmRun server',
|
description = 'Run the NilmRun server',
|
||||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
||||||
|
parser.add_argument("-v", "--version", action="version",
|
||||||
parser.add_argument("-V", "--version", action="version",
|
version=nilmrun.__version__)
|
||||||
version = nilmrun.__version__)
|
|
||||||
|
|
||||||
group = parser.add_argument_group("Standard options")
|
group = parser.add_argument_group("Standard options")
|
||||||
group.add_argument('-a', '--address',
|
group.add_argument('-a', '--address',
|
||||||
|
@ -36,25 +35,24 @@ def main():
|
||||||
embedded = False
|
embedded = False
|
||||||
server = nilmrun.server.Server(host = args.address,
|
server = nilmrun.server.Server(host = args.address,
|
||||||
port = args.port,
|
port = args.port,
|
||||||
embedded = embedded,
|
|
||||||
force_traceback = args.traceback)
|
force_traceback = args.traceback)
|
||||||
|
|
||||||
# Print info
|
# Print info
|
||||||
if not args.quiet:
|
if not args.quiet:
|
||||||
print "NilmRun version: %s" % nilmrun.__version__
|
print("NilmRun version: %s" % nilmrun.__version__)
|
||||||
print ("Note: This server does not do any authentication! " +
|
print(("Note: This server does not do any authentication! " +
|
||||||
"Anyone who can connect can run arbitrary commands.")
|
"Anyone who can connect can run arbitrary commands."))
|
||||||
if args.address == '0.0.0.0' or args.address == '::':
|
if args.address == '0.0.0.0' or args.address == '::':
|
||||||
host = socket.getfqdn()
|
host = socket.getfqdn()
|
||||||
else:
|
else:
|
||||||
host = args.address
|
host = args.address
|
||||||
print "Server URL: http://%s:%d/" % ( host, args.port)
|
print("Server URL: http://%s:%d/" % ( host, args.port))
|
||||||
print "----"
|
print("----")
|
||||||
|
|
||||||
server.start(blocking = True)
|
server.start(blocking = True)
|
||||||
|
|
||||||
if not args.quiet:
|
if not args.quiet:
|
||||||
print "Shutting down"
|
print("Shutting down")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
|
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
|
||||||
from nilmdb.utils.printf import *
|
from nilmdb.utils.printf import *
|
||||||
from nilmdb.utils import datetime_tz
|
import datetime_tz
|
||||||
import nilmrun
|
import nilmrun
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
@ -14,8 +14,8 @@ def main():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description = 'List NilmRun processes',
|
description = 'List NilmRun processes',
|
||||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
||||||
parser.add_argument("-V", "--version", action="version",
|
parser.add_argument("-v", "--version", action="version",
|
||||||
version = nilmrun.__version__)
|
version=nilmrun.__version__)
|
||||||
group = parser.add_argument_group("Standard options")
|
group = parser.add_argument_group("Standard options")
|
||||||
group.add_argument('-u', '--url',
|
group.add_argument('-u', '--url',
|
||||||
help = 'NilmRun server URL', default = def_url)
|
help = 'NilmRun server URL', default = def_url)
|
||||||
|
@ -40,13 +40,14 @@ def main():
|
||||||
round(info['system']['mem_used'] * 100.0
|
round(info['system']['mem_used'] * 100.0
|
||||||
/ info['system']['mem_total']))
|
/ info['system']['mem_total']))
|
||||||
|
|
||||||
|
# Print process detail for each managed process
|
||||||
|
fmt = "%-36s %-6s %-15s %-4s %-3s %-5s\n"
|
||||||
|
printf(fmt, "PID", "STATE", "SINCE", "PROC", "CPU", "LOG")
|
||||||
|
|
||||||
if len(info['pids']) == 0:
|
if len(info['pids']) == 0:
|
||||||
printf("No running processes\n")
|
printf("No running processes\n")
|
||||||
raise SystemExit(0)
|
raise SystemExit(0)
|
||||||
|
|
||||||
# Print process detail for each managed process
|
|
||||||
fmt = "%-36s %-6s %-15s %-4s %-3s %-5s\n"
|
|
||||||
printf(fmt, "PID", "STATE", "SINCE", "PROC", "CPU", "LOG")
|
|
||||||
for pid in sorted(info['pids'].keys()):
|
for pid in sorted(info['pids'].keys()):
|
||||||
pidinfo = client.get("process/status", { "pid": pid })
|
pidinfo = client.get("process/status", { "pid": pid })
|
||||||
if pidinfo['alive']:
|
if pidinfo['alive']:
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
|
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
|
||||||
from nilmdb.utils.printf import *
|
from nilmdb.utils.printf import *
|
||||||
|
@ -15,8 +15,8 @@ def main():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description = 'Run a command on the NilmRun server',
|
description = 'Run a command on the NilmRun server',
|
||||||
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
||||||
parser.add_argument("-V", "--version", action="version",
|
parser.add_argument("-v", "--version", action="version",
|
||||||
version = nilmrun.__version__)
|
version=nilmrun.__version__)
|
||||||
group = parser.add_argument_group("Standard options")
|
group = parser.add_argument_group("Standard options")
|
||||||
group.add_argument('-u', '--url',
|
group.add_argument('-u', '--url',
|
||||||
help = 'NilmRun server URL', default = def_url)
|
help = 'NilmRun server URL', default = def_url)
|
||||||
|
@ -31,14 +31,15 @@ def main():
|
||||||
help="Arguments for command")
|
help="Arguments for command")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
|
client = HTTPClient(baseurl=args.url, verify_ssl=not args.noverify,
|
||||||
|
post_json=True)
|
||||||
|
|
||||||
# Run command
|
# Run command
|
||||||
pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })
|
pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })
|
||||||
|
|
||||||
# If we're detaching, just print the PID
|
# If we're detaching, just print the PID
|
||||||
if args.detach:
|
if args.detach:
|
||||||
print pid
|
print(pid)
|
||||||
raise SystemExit(0)
|
raise SystemExit(0)
|
||||||
|
|
||||||
# Otherwise, watch the log output, and kill the process when it's done
|
# Otherwise, watch the log output, and kill the process when it's done
|
||||||
|
|
16
setup.cfg
16
setup.cfg
|
@ -20,3 +20,19 @@ cover-erase=1
|
||||||
stop=1
|
stop=1
|
||||||
verbosity=2
|
verbosity=2
|
||||||
tests=tests
|
tests=tests
|
||||||
|
|
||||||
|
[versioneer]
|
||||||
|
VCS=git
|
||||||
|
style=pep440
|
||||||
|
versionfile_source=nilmrun/_version.py
|
||||||
|
versionfile_build=nilmrun/_version.py
|
||||||
|
tag_prefix=nilmrun-
|
||||||
|
parentdir_prefix=nilmrun-
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
exclude=_version.py
|
||||||
|
extend-ignore=E731
|
||||||
|
|
||||||
|
[pylint]
|
||||||
|
ignore=_version.py
|
||||||
|
disable=C0103,C0111,R0913,R0914
|
||||||
|
|
53
setup.py
53
setup.py
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
# To release a new version, tag it:
|
# To release a new version, tag it:
|
||||||
# git tag -a nilmrun-1.1 -m "Version 1.1"
|
# git tag -a nilmrun-1.1 -m "Version 1.1"
|
||||||
|
@ -6,66 +6,31 @@
|
||||||
# Then just package it up:
|
# Then just package it up:
|
||||||
# python setup.py sdist
|
# python setup.py sdist
|
||||||
|
|
||||||
# This is supposed to be using Distribute:
|
|
||||||
#
|
|
||||||
# distutils provides a "setup" method.
|
|
||||||
# setuptools is a set of monkeypatches on top of that.
|
|
||||||
# distribute is a particular version/implementation of setuptools.
|
|
||||||
#
|
|
||||||
# So we don't really know if this is using the old setuptools or the
|
|
||||||
# Distribute-provided version of setuptools.
|
|
||||||
|
|
||||||
import traceback
|
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
|
from setuptools import setup
|
||||||
try:
|
|
||||||
from setuptools import setup, find_packages
|
|
||||||
import distutils.version
|
|
||||||
except ImportError:
|
|
||||||
traceback.print_exc()
|
|
||||||
print "Please install the prerequisites listed in README.txt"
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# Versioneer manages version numbers from git tags.
|
# Versioneer manages version numbers from git tags.
|
||||||
# https://github.com/warner/python-versioneer
|
# https://github.com/warner/python-versioneer
|
||||||
import versioneer
|
import versioneer
|
||||||
versioneer.versionfile_source = 'nilmrun/_version.py'
|
|
||||||
versioneer.versionfile_build = 'nilmrun/_version.py'
|
|
||||||
versioneer.tag_prefix = 'nilmrun-'
|
|
||||||
versioneer.parentdir_prefix = 'nilmrun-'
|
|
||||||
|
|
||||||
# Hack to workaround logging/multiprocessing issue:
|
# Get list of requirements to use in `install_requires` below. Note
|
||||||
# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
|
# that we don't make a distinction between things that are actually
|
||||||
try: import multiprocessing
|
# required for end-users vs developers (or use `test_requires` or
|
||||||
except: pass
|
# anything else) -- just install everything for simplicity.
|
||||||
|
install_requires = open('requirements.txt').readlines()
|
||||||
# We need a MANIFEST.in. Generate it here rather than polluting the
|
|
||||||
# repository with yet another setup-related file.
|
|
||||||
with open("MANIFEST.in", "w") as m:
|
|
||||||
m.write("""
|
|
||||||
# Root
|
|
||||||
include README.txt
|
|
||||||
include setup.py
|
|
||||||
include versioneer.py
|
|
||||||
include Makefile
|
|
||||||
""")
|
|
||||||
|
|
||||||
# Run setup
|
# Run setup
|
||||||
setup(name='nilmrun',
|
setup(name='nilmrun',
|
||||||
version = versioneer.get_version(),
|
version = versioneer.get_version(),
|
||||||
cmdclass = versioneer.get_cmdclass(),
|
cmdclass = versioneer.get_cmdclass(),
|
||||||
url = 'https://git.jim.sh/jim/lees/nilmrun.git',
|
url = 'https://git.jim.sh/nilm/nilmrun.git',
|
||||||
author = 'Jim Paris',
|
author = 'Jim Paris',
|
||||||
description = "NILM Database Filter Runner",
|
description = "NILM Database Filter Runner",
|
||||||
long_description = "NILM Database Filter Runner",
|
long_description = "NILM Database Filter Runner",
|
||||||
license = "Proprietary",
|
license = "Proprietary",
|
||||||
author_email = 'jim@jtan.com',
|
author_email = 'jim@jtan.com',
|
||||||
install_requires = [ 'nilmdb >= 1.9.5',
|
install_requires = install_requires,
|
||||||
'psutil >= 0.3.0',
|
|
||||||
'cherrypy >= 3.2',
|
|
||||||
'simplejson',
|
|
||||||
],
|
|
||||||
packages = [ 'nilmrun',
|
packages = [ 'nilmrun',
|
||||||
'nilmrun.scripts',
|
'nilmrun.scripts',
|
||||||
],
|
],
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import nose
|
import nose
|
||||||
import os
|
import os
|
||||||
|
|
|
@ -15,14 +15,15 @@ import distutils.version
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import cStringIO
|
import io
|
||||||
import simplejson as json
|
import json
|
||||||
import unittest
|
import unittest
|
||||||
import warnings
|
import warnings
|
||||||
import time
|
import time
|
||||||
import re
|
import re
|
||||||
import urllib2
|
import urllib.request, urllib.error, urllib.parse
|
||||||
from urllib2 import urlopen, HTTPError
|
from urllib.request import urlopen
|
||||||
|
from urllib.error import HTTPError
|
||||||
import requests
|
import requests
|
||||||
import pprint
|
import pprint
|
||||||
import textwrap
|
import textwrap
|
||||||
|
@ -128,10 +129,27 @@ class TestClient(object):
|
||||||
|
|
||||||
def _run_testfilter(self, client, args):
|
def _run_testfilter(self, client, args):
|
||||||
code = textwrap.dedent("""
|
code = textwrap.dedent("""
|
||||||
import nilmrun.testfilter
|
from nilmdb.utils.printf import *
|
||||||
import simplejson as json
|
import time
|
||||||
|
import signal
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
nilmrun.testfilter.test(json.loads(sys.argv[1]))
|
# This is just for testing the process management.
|
||||||
|
def test(n):
|
||||||
|
n = int(n)
|
||||||
|
if n < 0: # raise an exception
|
||||||
|
raise Exception("test exception")
|
||||||
|
if n == 0: # ignore SIGTERM and count to 100
|
||||||
|
n = 100
|
||||||
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||||
|
for x in range(n):
|
||||||
|
s = sprintf("dummy %d\\n", x)
|
||||||
|
if x & 1:
|
||||||
|
sys.stdout.write(s)
|
||||||
|
else:
|
||||||
|
sys.stderr.write(s)
|
||||||
|
time.sleep(0.1)
|
||||||
|
test(json.loads(sys.argv[1]))
|
||||||
""")
|
""")
|
||||||
jsonargs = json.dumps(args)
|
jsonargs = json.dumps(args)
|
||||||
return client.post("run/code", { "code": code, "args": [ jsonargs ] })
|
return client.post("run/code", { "code": code, "args": [ jsonargs ] })
|
||||||
|
@ -269,9 +287,9 @@ class TestClient(object):
|
||||||
|
|
||||||
# basic code snippet
|
# basic code snippet
|
||||||
code = textwrap.dedent("""
|
code = textwrap.dedent("""
|
||||||
print 'hello'
|
print('hello')
|
||||||
def foo(arg):
|
def foo(arg):
|
||||||
print 'world'
|
print('world')
|
||||||
""")
|
""")
|
||||||
status = do(code, [], False)
|
status = do(code, [], False)
|
||||||
eq_("hello\n", status["log"])
|
eq_("hello\n", status["log"])
|
||||||
|
@ -280,7 +298,7 @@ class TestClient(object):
|
||||||
# compile error
|
# compile error
|
||||||
code = textwrap.dedent("""
|
code = textwrap.dedent("""
|
||||||
def foo(arg:
|
def foo(arg:
|
||||||
print 'hello'
|
print('hello')
|
||||||
""")
|
""")
|
||||||
status = do(code, [], False)
|
status = do(code, [], False)
|
||||||
in_("SyntaxError", status["log"])
|
in_("SyntaxError", status["log"])
|
||||||
|
@ -305,11 +323,11 @@ class TestClient(object):
|
||||||
# argument handling (strings come in as unicode)
|
# argument handling (strings come in as unicode)
|
||||||
code = textwrap.dedent("""
|
code = textwrap.dedent("""
|
||||||
import sys
|
import sys
|
||||||
print sys.argv[1].encode('ascii'), sys.argv[2]
|
print(sys.argv[1], sys.argv[2])
|
||||||
sys.exit(0) # also test raising SystemExit
|
sys.exit(0) # also test raising SystemExit
|
||||||
""")
|
""")
|
||||||
with assert_raises(ClientError) as e:
|
with assert_raises(ClientError) as e:
|
||||||
do(code, ["hello", 123], False)
|
do(code, ["hello", 123], False)
|
||||||
in_("400 Bad Request", str(e.exception))
|
in_("400 Bad Request", str(e.exception))
|
||||||
status = do(code, ["hello", "123"], False)
|
status = do(code, ["hello", "123"], False)
|
||||||
eq_(status["log"], "hello 123\n")
|
eq_(status["log"], "hello 123\n")
|
||||||
|
@ -318,9 +336,9 @@ class TestClient(object):
|
||||||
# try killing a long-running process
|
# try killing a long-running process
|
||||||
code = textwrap.dedent("""
|
code = textwrap.dedent("""
|
||||||
import time
|
import time
|
||||||
print 'hello'
|
print('hello')
|
||||||
time.sleep(60)
|
time.sleep(60)
|
||||||
print 'world'
|
print('world')
|
||||||
""")
|
""")
|
||||||
status = do(code, [], True)
|
status = do(code, [], True)
|
||||||
eq_(status["log"], "hello\n")
|
eq_(status["log"], "hello\n")
|
||||||
|
@ -329,7 +347,7 @@ class TestClient(object):
|
||||||
# default arguments are empty
|
# default arguments are empty
|
||||||
code = textwrap.dedent("""
|
code = textwrap.dedent("""
|
||||||
import sys
|
import sys
|
||||||
print 'args:', len(sys.argv[1:])
|
print('args:', len(sys.argv[1:]))
|
||||||
""")
|
""")
|
||||||
status = do(code, None, False)
|
status = do(code, None, False)
|
||||||
eq_(status["log"], "args: 0\n")
|
eq_(status["log"], "args: 0\n")
|
||||||
|
@ -352,8 +370,10 @@ class TestClient(object):
|
||||||
# start some processes
|
# start some processes
|
||||||
a = client.post("run/command", { "argv": ["sleep","60"] } )
|
a = client.post("run/command", { "argv": ["sleep","60"] } )
|
||||||
b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } )
|
b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } )
|
||||||
c = client.post("run/command", { "argv": ["sh","-c","burnP5;true"] } )
|
c = client.post("run/command", { "argv": [
|
||||||
d = client.post("run/command", { "argv": ["burnP5" ] } )
|
"sh","-c","dd if=/dev/zero of=/dev/null;true"] } )
|
||||||
|
d = client.post("run/command", { "argv": [
|
||||||
|
"dd", "if=/dev/zero", "of=/dev/null" ] } )
|
||||||
|
|
||||||
info = client.get("process/info")
|
info = client.get("process/info")
|
||||||
eq_(info["pids"][a]["procs"], 1)
|
eq_(info["pids"][a]["procs"], 1)
|
||||||
|
@ -365,9 +385,13 @@ class TestClient(object):
|
||||||
lt_(20, info["pids"][c]["cpu_percent"])
|
lt_(20, info["pids"][c]["cpu_percent"])
|
||||||
lt_(80, info["system"]["cpu_percent"])
|
lt_(80, info["system"]["cpu_percent"])
|
||||||
|
|
||||||
time.sleep(2)
|
for x in range(10):
|
||||||
info = client.get("process/info")
|
time.sleep(1)
|
||||||
eq_(info["pids"][b]["procs"], 0)
|
info = client.get("process/info")
|
||||||
|
if info["pids"][b]["procs"] != 2:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise Exception("process B didn't die: " + str(info["pids"][b]))
|
||||||
|
|
||||||
# kill all processes
|
# kill all processes
|
||||||
for pid in client.get("process/list"):
|
for pid in client.get("process/list"):
|
||||||
|
@ -377,19 +401,20 @@ class TestClient(object):
|
||||||
client = HTTPClient(baseurl = testurl, post_json = True)
|
client = HTTPClient(baseurl = testurl, post_json = True)
|
||||||
eq_(client.get("process/list"), [])
|
eq_(client.get("process/list"), [])
|
||||||
def verify(cmd, result):
|
def verify(cmd, result):
|
||||||
pid = client.post("run/command", { "argv": [ "sh", "-c", cmd ] })
|
pid = client.post("run/command", { "argv": [
|
||||||
|
"/bin/bash", "-c", cmd ] })
|
||||||
eq_(client.get("process/list"), [pid])
|
eq_(client.get("process/list"), [pid])
|
||||||
status = self.wait_end(client, pid)
|
status = self.wait_end(client, pid)
|
||||||
eq_(result, status["log"])
|
eq_(result, status["log"])
|
||||||
|
|
||||||
# Unicode should work
|
# Unicode should work
|
||||||
verify("echo -n hello", "hello")
|
verify("echo -n hello", "hello")
|
||||||
verify(u"echo -n ☠", u"☠")
|
verify("echo -n ☠", "☠")
|
||||||
verify("echo -ne \\\\xe2\\\\x98\\\\xa0", u"☠")
|
verify("echo -ne \\\\xe2\\\\x98\\\\xa0", "☠")
|
||||||
|
|
||||||
# Programs that spit out invalid UTF-8 should get replacement
|
# Programs that spit out invalid UTF-8 should get replacement
|
||||||
# markers
|
# markers
|
||||||
verify("echo -ne \\\\xae", u"\ufffd")
|
verify("echo -ne \\\\xae", "\ufffd")
|
||||||
|
|
||||||
def test_client_11_atexit(self):
|
def test_client_11_atexit(self):
|
||||||
# Leave a directory and running process behind, for the atexit
|
# Leave a directory and running process behind, for the atexit
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# Just some helpers for test functions
|
# Just some helpers for test functions
|
||||||
|
|
||||||
def myrepr(x):
|
def myrepr(x):
|
||||||
if isinstance(x, basestring):
|
if isinstance(x, str):
|
||||||
return '"' + x + '"'
|
return '"' + x + '"'
|
||||||
else:
|
else:
|
||||||
return repr(x)
|
return repr(x)
|
||||||
|
|
2165
versioneer.py
2165
versioneer.py
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user