Freeze requirements

This module was only used to assist testing, but wasn't actually
available in a test environment where no copies of nilmrun have been
installed.  Just code the contents directly into the filters that the
tests are running.

.gitattributes

@@ -1 +1 @@
-src/_version.py export-subst
+nilmrun/_version.py export-subst

.gitignore

@@ -3,8 +3,8 @@ build/
 *.pyc
 dist/
 nilmrun.egg-info/
+.eggs/
 
 # This gets generated as needed by setup.py
-MANIFEST.in
 MANIFEST
 

MANIFEST.in

@@ -0,0 +1,8 @@
+# Root
+include README.md
+include setup.py
+include versioneer.py
+include Makefile
+
+# Version
+include nilmrun/_version.py

Makefile

@@ -1,48 +1,49 @@
 # By default, run the tests.
 all: test
 
-test2:
-	nilmrun/trainola.py data.js
-
 version:
-	python setup.py version
+	python3 setup.py version
 
 build:
-	python setup.py build_ext --inplace
+	python3 setup.py build_ext --inplace
 
 dist: sdist
 sdist:
-	python setup.py sdist
+	python3 setup.py sdist
 
 install:
-	python setup.py install
+	python3 setup.py install
 
 develop:
-	python setup.py develop
+	python3 setup.py develop
 
 docs:
 	make -C docs
 
+ctrl: flake
+flake:
+	flake8 nilmrun
 lint:
-	pylint --rcfile=.pylintrc nilmdb
+	pylint3 --rcfile=setup.cfg nilmrun
 
 test:
-ifeq ($(INSIDE_EMACS), t)
+ifneq ($(INSIDE_EMACS),)
 # Use the slightly more flexible script
-	python setup.py build_ext --inplace
+	python3 setup.py build_ext --inplace
-	python tests/runtests.py
+	python3 tests/runtests.py
 else
 # Let setup.py check dependencies, build stuff, and run the test
-	python setup.py nosetests
+	python3 setup.py nosetests
 endif
 
 clean::
+	find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
 	rm -f .coverage
-	find . -name '*pyc' | xargs rm -f
+	rm -rf nilmrun.egg-info/ build/
-	rm -rf nilmtools.egg-info/ build/ MANIFEST.in
 	make -C docs clean
 
 gitclean::
 	git clean -dXf
 
-.PHONY: all version dist sdist install docs lint test clean gitclean
+.PHONY: all version dist sdist install docs test
+.PHONY: ctrl lint flake clean gitclean

README.md

@@ -0,0 +1,32 @@
+# nilmrun: Run NilmDB filters
+by Jim Paris <jim@jtan.com>
+
+## Prerequisites:
+
+    # Runtime and build environments
+    sudo apt-get install python3
+
+    # Create a new Python virtual environment to isolate deps.
+    python3 -m venv ../venv
+    source ../venv/bin/activate   # run "deactivate" to leave
+
+    # Install all Python dependencies
+    pip3 install -r requirements.txt
+
+## Install:
+
+Install it into the virtual environment
+
+    python3 setup.py install
+
+If you want to instead install it system-wide, you will also need to
+install the requirements system-wide:
+
+    sudo pip3 install -r requirements.txt
+    sudo python3 setup.py install
+
+## Usage:
+
+    nilmrun-server --help
+
+See docs/wsgi.md for info on setting up a WSGI application in Apache.

README.txt

@@ -1,20 +0,0 @@
-nilmrun: Run NilmDB filters
-by Jim Paris <jim@jtan.com>
-
-Prerequisites:
-
-  # Runtime and build environments
-  sudo apt-get install python2.7 python-setuptools
-
-  # Plus nilmdb and its dependencies
-  nilmdb (1.8.3+)
-
-Install:
-
-  python setup.py install
-
-Usage:
-
-  nilmrun-server --help
-
-See docs/wsgi.md for info on setting up a WSGI application in Apache.

nilmrun/__init__.py

@@ -1,5 +1,3 @@
-import nilmrun.processmanager
-
 from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions

nilmrun/_version.py

@@ -1,197 +1,520 @@
 
-IN_LONG_VERSION_PY = True
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# versioneer-0.18 (https://github.com/warner/python-versioneer)
-
-# these strings will be replaced by git during git-archive
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
 
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
 
-def run_command(args, cwd=None, verbose=False):
+
-    try:
+def get_keywords():
-        # remember shell=False, so use git.cmd on windows, not just git
+    """Get the keywords needed to look up the version information."""
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
+    # these strings will be replaced by git during git-archive.
-    except EnvironmentError:
+    # setup.py/versioneer.py will grep for the variable names, so they must
-        e = sys.exc_info()[1]
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    git_date = "$Format:%ci$"
+    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = "nilmrun-"
+    cfg.parentdir_prefix = "nilmrun-"
+    cfg.versionfile_source = "nilmrun/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+                env=None):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+                                 stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None, None
+    else:
         if verbose:
-            print("unable to run %s" % args[0])
+            print("unable to find command, tried %s" % (commands,))
-            print(e)
+        return None, None
-        return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %s (error)" % args[0])
+            print("unable to run %s (error)" % dispcmd)
-        return None
+            print("stdout was %s" % stdout)
-    return stdout
+        return None, p.returncode
+    return stdout, p.returncode
 
 
-import sys
+def versions_from_parentdir(parentdir_prefix, root, verbose):
-import re
+    """Try to determine the version from the parent directory name.
-import os.path
 
-def get_expanded_variables(versionfile_source):
+    Source tarballs conventionally unpack into a directory that includes both
+    the project name and a version string. We will also support searching up
+    two directory levels for an appropriately named parent directory
+    """
+    rootdirs = []
+
+    for i in range(3):
+        dirname = os.path.basename(root)
+        if dirname.startswith(parentdir_prefix):
+            return {"version": dirname[len(parentdir_prefix):],
+                    "full-revisionid": None,
+                    "dirty": False, "error": None, "date": None}
+        else:
+            rootdirs.append(root)
+            root = os.path.dirname(root)  # up a level
+
+    if verbose:
+        print("Tried directories %s but none started with prefix %s" %
+              (str(rootdirs), parentdir_prefix))
+    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
+    # keywords. When used from setup.py, we don't want to import _version.py,
-    # _version.py, so we do it with a regexp instead. This function is not
+    # so we do it with a regexp instead. This function is not used from
-    # used from _version.py.
+    # _version.py.
-    variables = {}
+    keywords = {}
     try:
-        for line in open(versionfile_source,"r").readlines():
+        f = open(versionfile_abs, "r")
+        for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
+            if line.strip().startswith("git_date ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["date"] = mo.group(1)
+        f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
+
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    date = keywords.get("date")
+    if date is not None:
+        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+        # -like" string, which we must then edit to make compliant), because
+        # it's been around since git-1.5.3, and it's too difficult to
+        # discover which version we're using, or to work around using an
+        # older one.
+        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
+            print("keywords are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
-    for ref in list(refs):
+    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
-        if not re.search(r'\d', ref):
+    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
-            if verbose:
+    TAG = "tag: "
-                print("discarding '%s', no digits" % ref)
+    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
-            refs.discard(ref)
+    if not tags:
-            # Assume all version tags have a digit. git's %d expansion
+        # Either we're using git < 1.8.3, or there really are no tags. We use
-            # behaves like git log --decorate=short and strips out the
+        # a heuristic: assume all version tags have a digit. The old git %d
-            # refs/heads/ and refs/tags/ prefixes that would let us
+        # expansion behaves like git log --decorate=short and strips out the
-            # distinguish between branches and tags. By ignoring refnames
+        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
-            # without digits, we filter out many common branch names like
+        # between branches and tags. By ignoring refnames without digits, we
-            # "release" and "stabilization", as well as "HEAD" and "master".
+        # filter out many common branch names like "release" and
+        # "stabilization", as well as "HEAD" and "master".
+        tags = set([r for r in refs if re.search(r'\d', r)])
+        if verbose:
+            print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
-        print("remaining refs: %s" % ",".join(sorted(refs)))
+        print("likely tags: %s" % ",".join(sorted(tags)))
-    for ref in sorted(refs):
+    for ref in sorted(tags):
         # sorting will prefer e.g. "2.0" over "2.0rc1"
         if ref.startswith(tag_prefix):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %s" % r)
-            return { "version": r,
+            return {"version": r,
-                     "full": variables["full"].strip() }
+                    "full-revisionid": keywords["full"].strip(),
-    # no suitable tags, so we use the full revision id
+                    "dirty": False, "error": None,
+                    "date": date}
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
+        print("no suitable tags, using unknown + full revision id")
-    return { "version": variables["full"].strip(),
+    return {"version": "0+unknown",
-             "full": variables["full"].strip() }
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags", "date": None}
 
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
+
-    # this runs 'git' from the root of the source tree. That either means
+@register_vcs_handler("git", "pieces_from_vcs")
-    # someone ran a setup.py command (and this code is in versioneer.py, so
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
+    """Get version from 'git describe' in the root of the source tree.
-    # the source tree), or someone ran a project-specific entry point (and
+
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
+    This only gets called if the git-archive 'subst' keywords were *not*
-    # containing directory is somewhere deeper in the source tree). This only
+    expanded, and _version.py hasn't already been rewritten with a short
-    # gets called if the git-archive 'subst' variables were *not* expanded,
+    version string, meaning we're inside a checked out source tree.
-    # and _version.py hasn't already been rewritten with a short version
+    """
-    # string, meaning we're inside a checked out source tree.
+    GITS = ["git"]
+    if sys.platform == "win32":
+        GITS = ["git.cmd", "git.exe"]
+
+    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+                          hide_stderr=True)
+    if rc != 0:
+        if verbose:
+            print("Directory %s not under git control" % root)
+        raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+                                          "--always", "--long",
+                                          "--match", "%s*" % tag_prefix],
+                                   cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                    cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    # commit date: see ISO-8601 comment in git_versions_from_keywords()
+    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
+                       cwd=root)[0].strip()
+    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"],
+                "date": None}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None,
+            "date": pieces.get("date")}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
 
     try:
-        here = os.path.abspath(__file__)
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
-    except NameError:
+                                          verbose)
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
+    except NotThisMethod:
-        return {} # not always correct
+        pass
 
-    # versionfile_source is the relative path from the top of the source tree
+    try:
-    # (where the .git directory might live) to this file. Invert this to find
+        root = os.path.realpath(__file__)
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
-    if not os.path.exists(os.path.join(root, ".git")):
-        if verbose:
-            print("no .git in %s" % root)
-        return {}
-
-    GIT = "git"
-    if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
         # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
+        # tree (where the .git directory might live) to this file. Invert
-        root = here
+        # this to find the root from __file__.
-        for i in range(len(versionfile_source.split("/"))):
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
-    else:
+    except NameError:
-        # we're running from versioneer.py, which means we're running from
+        return {"version": "0+unknown", "full-revisionid": None,
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
+                "dirty": None,
-        here = os.path.abspath(sys.argv[0])
+                "error": "unable to find root of source tree",
-        root = os.path.dirname(here)
+                "date": None}
 
-    # Source tarballs conventionally unpack into a directory that includes
+    try:
-    # both the project name and a version string.
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
-    dirname = os.path.basename(root)
+        return render(pieces, cfg.style)
-    if not dirname.startswith(parentdir_prefix):
+    except NotThisMethod:
-        if verbose:
+        pass
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
 
-tag_prefix = "nilmrun-"
+    try:
-parentdir_prefix = "nilmrun-"
+        if cfg.parentdir_prefix:
-versionfile_source = "nilmrun/_version.py"
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
-
+    except NotThisMethod:
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
+        pass
-    variables = { "refnames": git_refnames, "full": git_full }
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
 
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version", "date": None}

nilmrun/processmanager.py

@@ -1,10 +1,8 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
-
-from nilmdb.utils.printf import *
 
 import threading
 import subprocess
-import cStringIO
+import io
 import sys
 import os
 import signal
@@ -15,16 +13,18 @@ import tempfile
 import atexit
 import shutil
 
+
 class ProcessError(Exception):
     pass
 
+
 class LogReceiver(object):
     """Spawn a thread that listens to a pipe for log messages,
     and stores them locally."""
     def __init__(self, pipe):
         self.pipe = pipe
-        self.log = cStringIO.StringIO()
+        self.log = io.BytesIO()
-        self.thread = threading.Thread(target = self.run)
+        self.thread = threading.Thread(target=self.run)
         self.thread.start()
 
     def run(self):
@@ -39,11 +39,12 @@ class LogReceiver(object):
         return self.log.getvalue()
 
     def clear(self):
-        self.log = cStringIO.StringIO()
+        self.log = io.BytesIO()
+
 
 class Process(object):
     """Spawn and manage a subprocess, and capture its output."""
-    def __init__(self, argv, tempfile = None):
+    def __init__(self, argv, tempfile=None):
         self.start_time = None
 
         # Use a pipe for communicating log data
@@ -55,9 +56,9 @@ class Process(object):
 
         # Spawn the new process
         try:
-            self._process = subprocess.Popen(args = argv, stdin = nullfd,
+            self._process = subprocess.Popen(args=argv, stdin=nullfd,
-                                             stdout = wpipe, stderr = wpipe,
+                                             stdout=wpipe, stderr=wpipe,
-                                             close_fds = True, cwd = "/tmp")
+                                             close_fds=True, cwd="/tmp")
         except (OSError, TypeError) as e:
             raise ProcessError(str(e))
         finally:
@@ -69,7 +70,7 @@ class Process(object):
         self.start_time = time.time()
         self.pid = str(uuid.uuid1(self._process.pid or 0))
 
-    def _join(self, timeout = 1.0):
+    def _join(self, timeout=1.0):
         start = time.time()
         while True:
             if self._process.poll() is not None:
@@ -78,7 +79,7 @@ class Process(object):
                 return False
             time.sleep(0.1)
 
-    def terminate(self, timeout = 1.0):
+    def terminate(self, timeout=1.0):
         """Terminate a process, and all of its children that are in the same
         process group."""
         try:
@@ -89,18 +90,24 @@ class Process(object):
             def getpgid(pid):
                 try:
                     return os.getpgid(pid)
-                except OSError: # pragma: no cover
+                except OSError:  # pragma: no cover
                     return None
 
+            def kill(pid, sig):
+                try:
+                    return os.kill(pid, sig)
+                except OSError:  # pragma: no cover
+                    return
+
             # Find all children
             group = getpgid(self._process.pid)
             main = psutil.Process(self._process.pid)
-            allproc = [ main ] + main.get_children(recursive = True)
+            allproc = [main] + main.children(recursive=True)
 
             # Kill with SIGTERM, if they're still in this process group
             for proc in allproc:
                 if getpgid(proc.pid) == group:
-                    os.kill(proc.pid, signal.SIGTERM)
+                    kill(proc.pid, signal.SIGTERM)
 
             # Wait for it to die again
             if self._join(timeout):
@@ -109,11 +116,11 @@ class Process(object):
             # One more try with SIGKILL
             for proc in allproc:
                 if getpgid(proc.pid) == group:
-                    os.kill(proc.pid, signal.SIGKILL)
+                    kill(proc.pid, signal.SIGKILL)
 
             # See if it worked
             return self._join(timeout)
-        except psutil.Error: # pragma: no cover (race condition)
+        except psutil.Error:  # pragma: no cover (race condition)
             return True
 
     def clear_log(self):
@@ -136,22 +143,22 @@ class Process(object):
         Call .get_info() about a second later."""
         try:
             main = psutil.Process(self._process.pid)
-            self._process_list = [ main ] + main.get_children(recursive = True)
+            self._process_list = [main] + main.children(recursive=True)
             for proc in self._process_list:
-                proc.get_cpu_percent(0)
+                proc.cpu_percent(0)
-        except psutil.Error: # pragma: no cover (race condition)
+        except psutil.Error:  # pragma: no cover (race condition)
-            self._process_list = [ ]
+            self._process_list = []
 
     @staticmethod
     def get_empty_info():
-        return { "cpu_percent": 0,
+        return {"cpu_percent": 0,
-                 "cpu_user": 0,
+                "cpu_user": 0,
-                 "cpu_sys": 0,
+                "cpu_sys": 0,
-                 "mem_phys": 0,
+                "mem_phys": 0,
-                 "mem_virt": 0,
+                "mem_virt": 0,
-                 "io_read": 0,
+                "io_read": 0,
-                 "io_write": 0,
+                "io_write": 0,
-                 "procs": 0 }
+                "procs": 0}
 
     def get_info(self):
         """Return a dictionary with info about the process CPU and memory
@@ -159,14 +166,14 @@ class Process(object):
         d = self.get_empty_info()
         for proc in self._process_list:
             try:
-                d["cpu_percent"] += proc.get_cpu_percent(0)
+                d["cpu_percent"] += proc.cpu_percent(0)
-                cpuinfo = proc.get_cpu_times()
+                cpuinfo = proc.cpu_times()
                 d["cpu_user"] += cpuinfo.user
                 d["cpu_sys"] += cpuinfo.system
-                meminfo = proc.get_memory_info()
+                meminfo = proc.memory_info()
                 d["mem_phys"] += meminfo.rss
                 d["mem_virt"] += meminfo.vms
-                ioinfo = proc.get_io_counters()
+                ioinfo = proc.io_counters()
                 d["io_read"] += ioinfo.read_bytes
                 d["io_write"] += ioinfo.write_bytes
                 d["procs"] += 1
@@ -174,16 +181,35 @@ class Process(object):
                 pass
         return d
 
+
 class ProcessManager(object):
     """Track and manage a collection of Process objects"""
     def __init__(self):
         self.processes = {}
-        self.tmpfiles = {}
+        self.tmpdirs = {}
-        self.tmpdir = tempfile.mkdtemp(prefix = "nilmrun-usercode-")
+        atexit.register(self._atexit)
-        atexit.register(shutil.rmtree, self.tmpdir)
+
+    def _cleanup_tmpdir(self, pid):
+        if pid in self.tmpdirs:
+            try:
+                shutil.rmtree(self.tmpdirs[pid])
+            except OSError:  # pragma: no cover
+                pass
+            del self.tmpdirs[pid]
+
+    def _atexit(self):
+        # Kill remaining processes, remove their dirs
+        for pid in list(self.processes.keys()):
+            try:
+                self.processes[pid].terminate()
+                del self.processes[pid]
+                shutil.rmtree(self.tmpdirs[pid])
+                del self.tmpdirs[pid]
+            except Exception:  # pragma: no cover
+                pass
 
     def __iter__(self):
-        return iter(self.processes.keys())
+        return iter(list(self.processes.keys()))
 
     def __getitem__(self, key):
         return self.processes[key]
@@ -193,15 +219,33 @@ class ProcessManager(object):
         executed.  The arguments, which must be strings, will be
         accessible in the code as sys.argv[1:]."""
         # The easiest way to do this, by far, is to just write the
-        # code to a file.
+        # code to a file.  Make a directory to put it in.
-        (fd, path) = tempfile.mkstemp(prefix = "nilmrun-usercode-",
+        tmpdir = tempfile.mkdtemp(prefix="nilmrun-usercode-")
-                                      suffix = ".py", dir=self.tmpdir)
+        try:
-        with os.fdopen(fd, 'w') as f:
+            # Write the code
-            f.write(code)
+            codepath = os.path.join(tmpdir, "usercode.py")
-        argv = [ sys.executable, "-B", "-s", "-u", path ] + args
+            with open(codepath, "w") as f:
-        pid = self.run_command(argv)
+                f.write(code)
-        self.tmpfiles[pid] = path
+            # Save the args too, for debugging purposes
-        return pid
+            with open(os.path.join(tmpdir, "args.txt"), "w") as f:
+                f.write(repr(args))
+
+            # Run the code
+            argv = [sys.executable, "-B", "-s", "-u", codepath] + args
+            pid = self.run_command(argv)
+
+            # Save the temp dir
+            self.tmpdirs[pid] = tmpdir
+            tmpdir = None  # Don't need to remove it anymore
+
+            return pid
+        finally:
+            # Clean up tempdir if we didn't finish
+            if tmpdir is not None:
+                try:
+                    shutil.rmtree(tmpdir)
+                except OSError:  # pragma: no cover
+                    pass
 
     def run_command(self, argv):
         """Execute a command line program"""
@@ -213,38 +257,32 @@ class ProcessManager(object):
         return self.processes[pid].terminate()
 
     def remove(self, pid):
-        if pid in self.tmpfiles:
+        self._cleanup_tmpdir(pid)
-            try:
-                os.unlink(self.tmpfiles[pid])
-            except OSError: # pragma: no cover
-                pass
-            del self.tmpfiles[pid]
         del self.processes[pid]
 
     def get_info(self):
         """Get info about all running PIDs"""
-        info = { "total" : Process.get_empty_info(),
+        info = {
-                 "pids" : {},
+            "total": Process.get_empty_info(),
-                 "system" : {}
+            "pids": {},
-                 }
+            "system": {}
+        }
 
         # Trigger CPU usage collection
         for pid in self:
             self[pid].get_info_prepare()
-        psutil.cpu_percent(0, percpu = True)
+        psutil.cpu_percent(0, percpu=True)
 
         # Give it some time
         time.sleep(1)
 
         # Retrieve info for system
         info["system"]["cpu_percent"] = sum(psutil.cpu_percent(0, percpu=True))
-        info["system"]["cpu_max"] = 100.0 * psutil.NUM_CPUS
+        info["system"]["cpu_max"] = 100.0 * psutil.cpu_count()
-        info["system"]["procs"] = len(psutil.get_pid_list())
+        info["system"]["procs"] = len(psutil.pids())
-        # psutil > 0.6.0's psutil.virtual_memory() would be better here,
+        meminfo = psutil.virtual_memory()
-        # but this should give the same info.
-        meminfo = psutil.phymem_usage()
         info["system"]["mem_total"] = meminfo.total
-        info["system"]["mem_used"] = int(meminfo.total * meminfo.percent / 100)
+        info["system"]["mem_used"] = meminfo.used
 
         # Retrieve info for each PID
         for pid in self:

nilmrun/server.py

@@ -1,19 +1,12 @@
 """CherryPy-based server for running NILM filters via HTTP"""
 
 import cherrypy
-import sys
 import os
 import socket
-import simplejson as json
 import traceback
-import time
 
-import nilmdb
+from nilmdb.utils.printf import sprintf
-from nilmdb.utils.printf import *
 from nilmdb.server.serverutil import (
-        chunked_response,
-        response_type,
-        workaround_cp_bug_1200,
         exception_to_httperror,
         CORS_allow,
         json_to_request_params,
@@ -24,11 +17,12 @@ from nilmdb.server.serverutil import (
         )
 from nilmdb.utils import serializer_proxy
 import nilmrun
-import nilmrun.testfilter
+import nilmrun.processmanager
 
 # Add CORS_allow tool
 cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
 
+
 # CherryPy apps
 class App(object):
     """Root application for NILM runner"""
@@ -55,6 +49,7 @@ class App(object):
     def version(self):
         return nilmrun.__version__
 
+
 class AppProcess(object):
 
     def __init__(self, manager):
@@ -76,7 +71,7 @@ class AppProcess(object):
     # /process/status
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def status(self, pid, clear = False):
+    def status(self, pid, clear=False):
         """Return status about a process.  If clear = True, also clear
         the log."""
         clear = bool_param(clear)
@@ -106,18 +101,19 @@ class AppProcess(object):
     @cherrypy.expose
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
-    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods=["POST"])
     def remove(self, pid):
         """Remove a process from the manager, killing it if necessary."""
         if pid not in self.manager:
             raise cherrypy.HTTPError("404 Not Found", "No such PID")
-        if not self.manager.terminate(pid): # pragma: no cover
+        if not self.manager.terminate(pid):  # pragma: no cover
             raise cherrypy.HTTPError("503 Service Unavailable",
                                      "Failed to stop process")
         status = self.process_status(pid)
         self.manager.remove(pid)
         return status
 
+
 class AppRun(object):
     def __init__(self, manager):
         self.manager = manager
@@ -127,7 +123,7 @@ class AppRun(object):
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(nilmrun.processmanager.ProcessError)
-    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods=["POST"])
     def command(self, argv):
         """Execute an arbitrary program on the server.  argv is a
         list of the program and its arguments: 'argv[0]' is the program
@@ -142,8 +138,8 @@ class AppRun(object):
     @cherrypy.tools.json_in()
     @cherrypy.tools.json_out()
     @exception_to_httperror(nilmrun.processmanager.ProcessError)
-    @cherrypy.tools.CORS_allow(methods = ["POST"])
+    @cherrypy.tools.CORS_allow(methods=["POST"])
-    def code(self, code, args = None):
+    def code(self, code, args=None):
         """Execute arbitrary Python code.  'code' is a formatted string.
         It will be run as if it were written into a Python file and
         executed.  'args' is a list of strings, and they are passed
@@ -156,23 +152,21 @@ class AppRun(object):
                                      "args must be a list of strings")
         return self.manager.run_code(code, args)
 
+
 class Server(object):
-    def __init__(self, host = '127.0.0.1', port = 8080,
+    def __init__(self, host='127.0.0.1', port=8080,
-                 embedded = True,         # hide diagnostics and output, etc
+                 force_traceback=False,  # include traceback in all errors
-                 force_traceback = False, # include traceback in all errors
+                 basepath='',            # base URL path for cherrypy.tree
-                 basepath = '',           # base URL path for cherrypy.tree
                  ):
-        self.embedded = embedded
 
         # Build up global server configuration
         cherrypy.config.update({
+            'environment': 'embedded',
             'server.socket_host': host,
             'server.socket_port': port,
             'engine.autoreload_on': False,
             'server.max_request_body_size': 8*1024*1024,
             })
-        if self.embedded:
-            cherrypy.config.update({ 'environment': 'embedded' })
 
         # Build up application specific configuration
         app_config = {}
@@ -181,23 +175,25 @@ class Server(object):
             })
 
         # Some default headers to just help identify that things are working
-        app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
+        app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
 
         # Set up Cross-Origin Resource Sharing (CORS) handler so we
         # can correctly respond to browsers' CORS preflight requests.
         # This also limits verbs to GET and HEAD by default.
-        app_config.update({ 'tools.CORS_allow.on': True,
+        app_config.update({
-                            'tools.CORS_allow.methods': ['GET', 'HEAD'] })
+            'tools.CORS_allow.on': True,
+            'tools.CORS_allow.methods': ['GET', 'HEAD']
+        })
 
         # Configure the 'json_in' tool to also allow other content-types
         # (like x-www-form-urlencoded), and to treat JSON as a dict that
         # fills requests.param.
-        app_config.update({ 'tools.json_in.force': False,
+        app_config.update({'tools.json_in.force': False,
-                            'tools.json_in.processor': json_to_request_params })
+                           'tools.json_in.processor': json_to_request_params})
 
         # Send tracebacks in error responses.  They're hidden by the
         # error_page function for client errors (code 400-499).
-        app_config.update({ 'request.show_tracebacks' : True })
+        app_config.update({'request.show_tracebacks': True})
         self.force_traceback = force_traceback
 
         # Patch CherryPy error handler to never pad out error messages.
@@ -210,11 +206,12 @@ class Server(object):
         manager = serializer_proxy(nilmrun.processmanager.ProcessManager)()
 
         # Build up the application and mount it
+        self._manager = manager
         root = App()
         root.process = AppProcess(manager)
         root.run = AppRun(manager)
         cherrypy.tree.apps = {}
-        cherrypy.tree.mount(root, basepath, config = { "/" : app_config })
+        cherrypy.tree.mount(root, basepath, config={"/": app_config})
 
         # Set up the WSGI application pointer for external programs
         self.wsgi_application = cherrypy.tree
@@ -224,17 +221,20 @@ class Server(object):
         return json_error_page(status, message, traceback, version,
                                self.force_traceback)
 
-    def start(self, blocking = False, event = None):
+    def start(self, blocking=False, event=None):
-        cherrypy_start(blocking, event, self.embedded)
+        cherrypy_start(blocking, event)
 
     def stop(self):
         cherrypy_stop()
 
+
 # Multiple processes and threads should be OK here, but we'll still
 # follow the NilmDB approach of having just one globally initialized
 # copy of the server object.
 _wsgi_server = None
-def wsgi_application(basepath): # pragma: no cover
+
+
+def wsgi_application(basepath):  # pragma: no cover
     """Return a WSGI application object.
 
     'basepath' is the URL path of the application base, which
@@ -247,13 +247,11 @@ def wsgi_application(basepath): # pragma: no cover
             # Try to start the server
             try:
                 _wsgi_server = nilmrun.server.Server(
-                    embedded = True,
+                    basepath=basepath.rstrip('/'))
-                    basepath = basepath.rstrip('/'))
             except Exception:
                 # Build an error message on failure
                 import pprint
-                err = sprintf("Initializing nilmrun failed:\n\n",
+                err = "Initializing nilmrun failed:\n\n"
-                              dbpath)
                 err += traceback.format_exc()
                 try:
                     import pwd
@@ -268,8 +266,10 @@ def wsgi_application(basepath): # pragma: no cover
                 err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
         if _wsgi_server is None:
             # Serve up the error with our own mini WSGI app.
-            headers = [ ('Content-type', 'text/plain'),
+            headers = [
-                        ('Content-length', str(len(err))) ]
+                ('Content-type', 'text/plain'),
+                ('Content-length', str(len(err)))
+            ]
             start_response("500 Internal Server Error", headers)
             return [err]
 

nilmrun/testfilter.py

@@ -1,22 +0,0 @@
-#!/usr/bin/python
-
-from nilmdb.utils.printf import *
-import time
-import signal
-import sys
-
-# This is just for testing the process management.
-def test(n):
-    n = int(n)
-    if n < 0: # raise an exception
-        raise Exception("test exception")
-    if n == 0: # ignore SIGTERM and count to 100
-        n = 100
-        signal.signal(signal.SIGTERM, signal.SIG_IGN)
-    for x in range(n):
-        s = sprintf("dummy %d\n", x)
-        if x & 1:
-            sys.stdout.write(s)
-        else:
-            sys.stderr.write(s)
-        time.sleep(0.1)

requirements.txt

@@ -0,0 +1,2 @@
+nilmdb>=2.0.3
+psutil==5.7.2

scripts/kill.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
+from nilmdb.utils.printf import *
+import nilmrun
+
+import argparse
+import os
+import sys
+
+def main():
+    """Kill/remove a process from the NilmRun server"""
+    def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
+    parser = argparse.ArgumentParser(
+        description = 'Kill/remove a process from the NilmRun server',
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-v", "--version", action="version",
+                        version=nilmrun.__version__)
+    group = parser.add_argument_group("Standard options")
+    group.add_argument('-u', '--url',
+                       help = 'NilmRun server URL', default = def_url)
+    group.add_argument('-n', '--noverify', action="store_true",
+                       help = 'Disable SSL certificate verification')
+    group = parser.add_argument_group("Program")
+    group.add_argument('-q', '--quiet', action="store_true",
+                       help = "Don't print out the final log contents")
+    group.add_argument('pid', nargs='+', help="PIDs to kill")
+    args = parser.parse_args()
+
+    client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
+
+    # Kill or remove process
+    all_failed = True
+    for pid in args.pid:
+        try:
+            s = client.post("process/remove", { "pid": pid })
+            if not args.quiet:
+                sys.stdout.write(s['log'])
+            all_failed = False
+        except ClientError as e:
+            if "404" in e.status:
+                fprintf(sys.stderr, "no such pid: %s\n", pid)
+            else:
+                raise
+
+    # Return error if we failed to remove any of them
+    if all_failed:
+        raise SystemExit(1)
+
+if __name__ == "__main__":
+    main()

scripts/nilmrun_server.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import nilmrun.server
 import argparse
@@ -11,9 +11,8 @@ def main():
     parser = argparse.ArgumentParser(
         description = 'Run the NilmRun server',
         formatter_class = argparse.ArgumentDefaultsHelpFormatter)
-
+    parser.add_argument("-v", "--version", action="version",
-    parser.add_argument("-V", "--version", action="version",
+                        version=nilmrun.__version__)
-                        version = nilmrun.__version__)
 
     group = parser.add_argument_group("Standard options")
     group.add_argument('-a', '--address',
@@ -36,25 +35,24 @@ def main():
         embedded = False
     server = nilmrun.server.Server(host = args.address,
                                    port = args.port,
-                                   embedded = embedded,
                                    force_traceback = args.traceback)
 
     # Print info
     if not args.quiet:
-        print "NilmRun version: %s" % nilmrun.__version__
+        print("NilmRun version: %s" % nilmrun.__version__)
-        print ("Note: This server does not do any authentication! " +
+        print(("Note: This server does not do any authentication! " +
-               "Anyone who can connect can run arbitrary commands.")
+               "Anyone who can connect can run arbitrary commands."))
         if args.address == '0.0.0.0' or args.address == '::':
             host = socket.getfqdn()
         else:
             host = args.address
-        print "Server URL: http://%s:%d/" % ( host, args.port)
+        print("Server URL: http://%s:%d/" % ( host, args.port))
-        print "----"
+        print("----")
 
     server.start(blocking = True)
 
     if not args.quiet:
-        print "Shutting down"
+        print("Shutting down")
 
 if __name__ == "__main__":
     main()

scripts/ps.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
+from nilmdb.utils.printf import *
+import datetime_tz
+import nilmrun
+
+import argparse
+import os
+
+def main():
+    """List NilmRun processes"""
+    def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
+    parser = argparse.ArgumentParser(
+        description = 'List NilmRun processes',
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-v", "--version", action="version",
+                        version=nilmrun.__version__)
+    group = parser.add_argument_group("Standard options")
+    group.add_argument('-u', '--url',
+                       help = 'NilmRun server URL', default = def_url)
+    group.add_argument('-n', '--noverify', action="store_true",
+                       help = 'Disable SSL certificate verification')
+    args = parser.parse_args()
+
+    client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
+    # Print overall system info
+    info = client.get("process/info")
+    total = info['total']
+    system = info['system']
+    printf(" procs: %d nilm, %d other\n", info['total']['procs'],
+           info['system']['procs'] - info['total']['procs'])
+    printf("   cpu: %d%% nilm, %d%% other, %d%% max\n",
+           round(info['total']['cpu_percent']),
+           round(info['system']['cpu_percent'] - info['total']['cpu_percent']),
+           round(info['system']['cpu_max']))
+    printf("   mem: %d MiB used, %d MiB total, %d%%\n",
+           round(info['system']['mem_used'] / 1048576.0),
+           round(info['system']['mem_total'] / 1048576.0),
+           round(info['system']['mem_used'] * 100.0
+                 / info['system']['mem_total']))
+
+    # Print process detail for each managed process
+    fmt = "%-36s %-6s %-15s %-4s %-3s %-5s\n"
+    printf(fmt, "PID", "STATE", "SINCE", "PROC", "CPU", "LOG")
+
+    if len(info['pids']) == 0:
+        printf("No running processes\n")
+        raise SystemExit(0)
+
+    for pid in sorted(info['pids'].keys()):
+        pidinfo = client.get("process/status", { "pid": pid })
+        if pidinfo['alive']:
+            status = "alive"
+        else:
+            if pidinfo['exitcode']:
+                status = "error"
+            else:
+                status = "done"
+        dt = datetime_tz.datetime_tz.fromtimestamp(pidinfo['start_time'])
+        since = dt.strftime("%m/%d-%H:%M:%S")
+        printf(fmt, pid, status, since, info['pids'][pid]['procs'],
+               str(int(round(info['pids'][pid]['cpu_percent']))),
+               len(pidinfo['log']))
+
+if __name__ == "__main__":
+    main()

scripts/run.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
+from nilmdb.utils.printf import *
+import nilmrun
+
+import argparse
+import os
+import time
+import sys
+
+def main():
+    """Run a command on the NilmRun server"""
+    def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
+    parser = argparse.ArgumentParser(
+        description = 'Run a command on the NilmRun server',
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-v", "--version", action="version",
+                        version=nilmrun.__version__)
+    group = parser.add_argument_group("Standard options")
+    group.add_argument('-u', '--url',
+                       help = 'NilmRun server URL', default = def_url)
+    group.add_argument('-n', '--noverify', action="store_true",
+                       help = 'Disable SSL certificate verification')
+    group = parser.add_argument_group("Program")
+    group.add_argument('-d', '--detach', action="store_true",
+                       help = 'Run process and return immediately without '
+                       'printing its output')
+    group.add_argument('cmd', help="Command to run")
+    group.add_argument('arg', nargs=argparse.REMAINDER,
+                       help="Arguments for command")
+    args = parser.parse_args()
+
+    client = HTTPClient(baseurl=args.url, verify_ssl=not args.noverify,
+                        post_json=True)
+
+    # Run command
+    pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })
+
+    # If we're detaching, just print the PID
+    if args.detach:
+        print(pid)
+        raise SystemExit(0)
+
+    # Otherwise, watch the log output, and kill the process when it's done
+    # or when this script terminates.
+    try:
+        while True:
+            s = client.get("process/status", { "pid": pid, "clear": 1 })
+            sys.stdout.write(s['log'])
+            sys.stdout.flush()
+            if not s['alive']:
+                break
+            time.sleep(1)
+    finally:
+        s = client.post("process/remove", { "pid": pid })
+
+    raise SystemExit(s['exitcode'])
+
+if __name__ == "__main__":
+    main()

setup.cfg

@@ -20,3 +20,19 @@ cover-erase=1
 stop=1
 verbosity=2
 tests=tests
+
+[versioneer]
+VCS=git
+style=pep440
+versionfile_source=nilmrun/_version.py
+versionfile_build=nilmrun/_version.py
+tag_prefix=nilmrun-
+parentdir_prefix=nilmrun-
+
+[flake8]
+exclude=_version.py
+extend-ignore=E731
+
+[pylint]
+ignore=_version.py
+disable=C0103,C0111,R0913,R0914

setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 # To release a new version, tag it:
 #   git tag -a nilmrun-1.1 -m "Version 1.1"
@@ -6,66 +6,31 @@
 # Then just package it up:
 #   python setup.py sdist
 
-# This is supposed to be using Distribute:
-#
-#   distutils provides a "setup" method.
-#   setuptools is a set of monkeypatches on top of that.
-#   distribute is a particular version/implementation of setuptools.
-#
-# So we don't really know if this is using the old setuptools or the
-# Distribute-provided version of setuptools.
-
-import traceback
 import sys
 import os
-
+from setuptools import setup
-try:
-    from setuptools import setup, find_packages
-    import distutils.version
-except ImportError:
-    traceback.print_exc()
-    print "Please install the prerequisites listed in README.txt"
-    sys.exit(1)
 
 # Versioneer manages version numbers from git tags.
 # https://github.com/warner/python-versioneer
 import versioneer
-versioneer.versionfile_source = 'nilmrun/_version.py'
-versioneer.versionfile_build = 'nilmrun/_version.py'
-versioneer.tag_prefix = 'nilmrun-'
-versioneer.parentdir_prefix = 'nilmrun-'
 
-# Hack to workaround logging/multiprocessing issue:
+# Get list of requirements to use in `install_requires` below.  Note
-# https://groups.google.com/d/msg/nose-users/fnJ-kAUbYHQ/_UsLN786ygcJ
+# that we don't make a distinction between things that are actually
-try: import multiprocessing
+# required for end-users vs developers (or use `test_requires` or
-except: pass
+# anything else) -- just install everything for simplicity.
-
+install_requires = open('requirements.txt').readlines()
-# We need a MANIFEST.in.  Generate it here rather than polluting the
-# repository with yet another setup-related file.
-with open("MANIFEST.in", "w") as m:
-    m.write("""
-# Root
-include README.txt
-include setup.py
-include versioneer.py
-include Makefile
-""")
 
 # Run setup
 setup(name='nilmrun',
       version = versioneer.get_version(),
       cmdclass = versioneer.get_cmdclass(),
-      url = 'https://git.jim.sh/jim/lees/nilmrun.git',
+      url = 'https://git.jim.sh/nilm/nilmrun.git',
       author = 'Jim Paris',
       description = "NILM Database Filter Runner",
       long_description = "NILM Database Filter Runner",
       license = "Proprietary",
       author_email = 'jim@jtan.com',
-      install_requires = [ 'nilmdb >= 1.8.3',
+      install_requires = install_requires,
-                           'psutil >= 0.3.0',
-                           'cherrypy >= 3.2',
-                           'simplejson',
-                           ],
       packages = [ 'nilmrun',
                    'nilmrun.scripts',
                    ],
@@ -75,6 +40,9 @@ setup(name='nilmrun',
       entry_points = {
           'console_scripts': [
               'nilmrun-server = nilmrun.scripts.nilmrun_server:main',
+              'nilmrun-ps = nilmrun.scripts.ps:main',
+              'nilmrun-run = nilmrun.scripts.run:main',
+              'nilmrun-kill = nilmrun.scripts.kill:main',
               ],
           },
       zip_safe = False,

tests/runtests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import nose
 import os

tests/test_nilmrun.py

@@ -15,14 +15,15 @@ import distutils.version
 import os
 import sys
 import threading
-import cStringIO
+import io
-import simplejson as json
+import json
 import unittest
 import warnings
 import time
 import re
-import urllib2
+import urllib.request, urllib.error, urllib.parse
-from urllib2 import urlopen, HTTPError
+from urllib.request import urlopen
+from urllib.error import HTTPError
 import requests
 import pprint
 import textwrap
@@ -128,10 +129,27 @@ class TestClient(object):
 
     def _run_testfilter(self, client, args):
         code = textwrap.dedent("""
-        import nilmrun.testfilter
+        from nilmdb.utils.printf import *
-        import simplejson as json
+        import time
+        import signal
+        import json
         import sys
-        nilmrun.testfilter.test(json.loads(sys.argv[1]))
+        # This is just for testing the process management.
+        def test(n):
+            n = int(n)
+            if n < 0: # raise an exception
+                raise Exception("test exception")
+            if n == 0: # ignore SIGTERM and count to 100
+                n = 100
+                signal.signal(signal.SIGTERM, signal.SIG_IGN)
+            for x in range(n):
+                s = sprintf("dummy %d\\n", x)
+                if x & 1:
+                    sys.stdout.write(s)
+                else:
+                    sys.stderr.write(s)
+                time.sleep(0.1)
+        test(json.loads(sys.argv[1]))
         """)
         jsonargs = json.dumps(args)
         return client.post("run/code", { "code": code, "args": [ jsonargs ] })
@@ -269,9 +287,9 @@ class TestClient(object):
 
         # basic code snippet
         code = textwrap.dedent("""
-        print 'hello'
+        print('hello')
         def foo(arg):
-            print 'world'
+            print('world')
         """)
         status = do(code, [], False)
         eq_("hello\n", status["log"])
@@ -280,7 +298,7 @@ class TestClient(object):
         # compile error
         code = textwrap.dedent("""
         def foo(arg:
-            print 'hello'
+            print('hello')
         """)
         status = do(code, [], False)
         in_("SyntaxError", status["log"])
@@ -305,11 +323,11 @@ class TestClient(object):
         # argument handling (strings come in as unicode)
         code = textwrap.dedent("""
         import sys
-        print sys.argv[1].encode('ascii'), sys.argv[2]
+        print(sys.argv[1], sys.argv[2])
         sys.exit(0)  # also test raising SystemExit
         """)
         with assert_raises(ClientError) as e:
-	    do(code, ["hello", 123], False)
+            do(code, ["hello", 123], False)
         in_("400 Bad Request", str(e.exception))
         status = do(code, ["hello", "123"], False)
         eq_(status["log"], "hello 123\n")
@@ -318,9 +336,9 @@ class TestClient(object):
         # try killing a long-running process
         code = textwrap.dedent("""
         import time
-        print 'hello'
+        print('hello')
         time.sleep(60)
-        print 'world'
+        print('world')
         """)
         status = do(code, [], True)
         eq_(status["log"], "hello\n")
@@ -329,7 +347,7 @@ class TestClient(object):
         # default arguments are empty
         code = textwrap.dedent("""
         import sys
-        print 'args:', len(sys.argv[1:])
+        print('args:', len(sys.argv[1:]))
         """)
         status = do(code, None, False)
         eq_(status["log"], "args: 0\n")
@@ -352,8 +370,10 @@ class TestClient(object):
         # start some processes
         a = client.post("run/command", { "argv": ["sleep","60"] } )
         b = client.post("run/command", { "argv": ["sh","-c","sleep 2;true"] } )
-        c = client.post("run/command", { "argv": ["sh","-c","burnP5;true"] } )
+        c = client.post("run/command", { "argv": [
-        d = client.post("run/command", { "argv": ["burnP5" ] } )
+            "sh","-c","dd if=/dev/zero of=/dev/null;true"] } )
+        d = client.post("run/command", { "argv": [
+            "dd", "if=/dev/zero", "of=/dev/null" ] } )
 
         info = client.get("process/info")
         eq_(info["pids"][a]["procs"], 1)
@@ -365,9 +385,13 @@ class TestClient(object):
         lt_(20, info["pids"][c]["cpu_percent"])
         lt_(80, info["system"]["cpu_percent"])
 
-        time.sleep(2)
+        for x in range(10):
-        info = client.get("process/info")
+            time.sleep(1)
-        eq_(info["pids"][b]["procs"], 0)
+            info = client.get("process/info")
+            if info["pids"][b]["procs"] != 2:
+                break
+        else:
+            raise Exception("process B didn't die: " + str(info["pids"][b]))
 
         # kill all processes
         for pid in client.get("process/list"):
@@ -377,16 +401,34 @@ class TestClient(object):
         client = HTTPClient(baseurl = testurl, post_json = True)
         eq_(client.get("process/list"), [])
         def verify(cmd, result):
-            pid = client.post("run/command", { "argv": [ "sh", "-c", cmd ] })
+            pid = client.post("run/command", { "argv": [
+                "/bin/bash", "-c", cmd ] })
             eq_(client.get("process/list"), [pid])
             status = self.wait_end(client, pid)
             eq_(result, status["log"])
 
         # Unicode should work
         verify("echo -n hello", "hello")
-        verify(u"echo -n ☠", u"☠")
+        verify("echo -n ☠", "☠")
-        verify("echo -ne \\\\xe2\\\\x98\\\\xa0", u"☠")
+        verify("echo -ne \\\\xe2\\\\x98\\\\xa0", "☠")
 
         # Programs that spit out invalid UTF-8 should get replacement
         # markers
-        verify("echo -ne \\\\xae", u"\ufffd")
+        verify("echo -ne \\\\xae", "\ufffd")
+
+    def test_client_11_atexit(self):
+        # Leave a directory and running process behind, for the atexit
+        # handler to clean up.  Here we trigger the atexit manually,
+        # since it's hard to trigger it as part of the test suite.
+        client = HTTPClient(baseurl = testurl, post_json = True)
+        code = textwrap.dedent("""
+        import time
+        time.sleep(10)
+        """)
+        client.post("run/code", { "code": code, "args": [ "hello"] })
+
+        # Trigger atexit function
+        test_server._manager._atexit()
+
+        # Ensure no processes exit
+        eq_(client.get("process/list"), [])

tests/testutil/helpers.py

@@ -1,7 +1,7 @@
 # Just some helpers for test functions
 
 def myrepr(x):
-    if isinstance(x, basestring):
+    if isinstance(x, str):
         return '"' + x + '"'
     else:
         return repr(x)