Compare commits

...

6 Commits

17 changed files with 138 additions and 111 deletions

2
.gitattributes vendored
View File

@ -1 +1 @@
src/_version.py export-subst nilmrun/_version.py export-subst

2
.gitignore vendored
View File

@ -3,8 +3,8 @@ build/
*.pyc *.pyc
dist/ dist/
nilmrun.egg-info/ nilmrun.egg-info/
.eggs/
# This gets generated as needed by setup.py # This gets generated as needed by setup.py
MANIFEST.in
MANIFEST MANIFEST

8
MANIFEST.in Normal file
View File

@ -0,0 +1,8 @@
# Root
include README.md
include setup.py
include versioneer.py
include Makefile
# Version
include nilmrun/_version.py

View File

@ -1,9 +1,6 @@
# By default, run the tests. # By default, run the tests.
all: test all: test
test2:
nilmrun/trainola.py data.js
version: version:
python3 setup.py version python3 setup.py version
@ -23,8 +20,11 @@ develop:
docs: docs:
make -C docs make -C docs
ctrl: flake
flake:
flake8 nilmrun
lint: lint:
pylint3 --rcfile=.pylintrc nilmdb pylint3 --rcfile=setup.cfg nilmrun
test: test:
ifneq ($(INSIDE_EMACS),) ifneq ($(INSIDE_EMACS),)
@ -37,12 +37,13 @@ else
endif endif
clean:: clean::
find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
rm -f .coverage rm -f .coverage
find . -name '*pyc' | xargs rm -f rm -rf nilmrun.egg-info/ build/
rm -rf nilmtools.egg-info/ build/ MANIFEST.in
make -C docs clean make -C docs clean
gitclean:: gitclean::
git clean -dXf git clean -dXf
.PHONY: all version dist sdist install docs lint test clean gitclean .PHONY: all version dist sdist install docs test
.PHONY: ctrl lint flake clean gitclean

View File

@ -1,5 +1,3 @@
import nilmrun.processmanager
from ._version import get_versions from ._version import get_versions
__version__ = get_versions()['version'] __version__ = get_versions()['version']
del get_versions del get_versions

View File

@ -41,9 +41,9 @@ def get_config():
cfg = VersioneerConfig() cfg = VersioneerConfig()
cfg.VCS = "git" cfg.VCS = "git"
cfg.style = "pep440" cfg.style = "pep440"
cfg.tag_prefix = "nilmdb-" cfg.tag_prefix = "nilmrun-"
cfg.parentdir_prefix = "nilmdb-" cfg.parentdir_prefix = "nilmrun-"
cfg.versionfile_source = "nilmdb/_version.py" cfg.versionfile_source = "nilmrun/_version.py"
cfg.verbose = False cfg.verbose = False
return cfg return cfg
@ -86,20 +86,20 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
if e.errno == errno.ENOENT: if e.errno == errno.ENOENT:
continue continue
if verbose: if verbose:
print(("unable to run %s" % dispcmd)) print("unable to run %s" % dispcmd)
print(e) print(e)
return None, None return None, None
else: else:
if verbose: if verbose:
print(("unable to find command, tried %s" % (commands,))) print("unable to find command, tried %s" % (commands,))
return None, None return None, None
stdout = p.communicate()[0].strip() stdout = p.communicate()[0].strip()
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
stdout = stdout.decode() stdout = stdout.decode()
if p.returncode != 0: if p.returncode != 0:
if verbose: if verbose:
print(("unable to run %s (error)" % dispcmd)) print("unable to run %s (error)" % dispcmd)
print(("stdout was %s" % stdout)) print("stdout was %s" % stdout)
return None, p.returncode return None, p.returncode
return stdout, p.returncode return stdout, p.returncode
@ -124,8 +124,8 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
root = os.path.dirname(root) # up a level root = os.path.dirname(root) # up a level
if verbose: if verbose:
print(("Tried directories %s but none started with prefix %s" % print("Tried directories %s but none started with prefix %s" %
(str(rootdirs), parentdir_prefix))) (str(rootdirs), parentdir_prefix))
raise NotThisMethod("rootdir doesn't start with parentdir_prefix") raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
@ -192,15 +192,15 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
# "stabilization", as well as "HEAD" and "master". # "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)]) tags = set([r for r in refs if re.search(r'\d', r)])
if verbose: if verbose:
print(("discarding '%s', no digits" % ",".join(refs - tags))) print("discarding '%s', no digits" % ",".join(refs - tags))
if verbose: if verbose:
print(("likely tags: %s" % ",".join(sorted(tags)))) print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(tags): for ref in sorted(tags):
# sorting will prefer e.g. "2.0" over "2.0rc1" # sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix): if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):] r = ref[len(tag_prefix):]
if verbose: if verbose:
print(("picking %s" % r)) print("picking %s" % r)
return {"version": r, return {"version": r,
"full-revisionid": keywords["full"].strip(), "full-revisionid": keywords["full"].strip(),
"dirty": False, "error": None, "dirty": False, "error": None,
@ -229,7 +229,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
hide_stderr=True) hide_stderr=True)
if rc != 0: if rc != 0:
if verbose: if verbose:
print(("Directory %s not under git control" % root)) print("Directory %s not under git control" % root)
raise NotThisMethod("'git rev-parse --git-dir' returned error") raise NotThisMethod("'git rev-parse --git-dir' returned error")
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
@ -278,7 +278,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
if not full_tag.startswith(tag_prefix): if not full_tag.startswith(tag_prefix):
if verbose: if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'" fmt = "tag '%s' doesn't start with prefix '%s'"
print((fmt % (full_tag, tag_prefix))) print(fmt % (full_tag, tag_prefix))
pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
% (full_tag, tag_prefix)) % (full_tag, tag_prefix))
return pieces return pieces

View File

@ -1,6 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.utils.printf import *
import threading import threading
import subprocess import subprocess
@ -15,9 +13,11 @@ import tempfile
import atexit import atexit
import shutil import shutil
class ProcessError(Exception): class ProcessError(Exception):
pass pass
class LogReceiver(object): class LogReceiver(object):
"""Spawn a thread that listens to a pipe for log messages, """Spawn a thread that listens to a pipe for log messages,
and stores them locally.""" and stores them locally."""
@ -41,6 +41,7 @@ class LogReceiver(object):
def clear(self): def clear(self):
self.log = io.BytesIO() self.log = io.BytesIO()
class Process(object): class Process(object):
"""Spawn and manage a subprocess, and capture its output.""" """Spawn and manage a subprocess, and capture its output."""
def __init__(self, argv, tempfile=None): def __init__(self, argv, tempfile=None):
@ -180,6 +181,7 @@ class Process(object):
pass pass
return d return d
class ProcessManager(object): class ProcessManager(object):
"""Track and manage a collection of Process objects""" """Track and manage a collection of Process objects"""
def __init__(self): def __init__(self):
@ -260,7 +262,8 @@ class ProcessManager(object):
def get_info(self): def get_info(self):
"""Get info about all running PIDs""" """Get info about all running PIDs"""
info = { "total" : Process.get_empty_info(), info = {
"total": Process.get_empty_info(),
"pids": {}, "pids": {},
"system": {} "system": {}
} }

View File

@ -1,18 +1,12 @@
"""CherryPy-based server for running NILM filters via HTTP""" """CherryPy-based server for running NILM filters via HTTP"""
import cherrypy import cherrypy
import sys
import os import os
import socket import socket
import json
import traceback import traceback
import time
import nilmdb from nilmdb.utils.printf import sprintf
from nilmdb.utils.printf import *
from nilmdb.server.serverutil import ( from nilmdb.server.serverutil import (
chunked_response,
response_type,
exception_to_httperror, exception_to_httperror,
CORS_allow, CORS_allow,
json_to_request_params, json_to_request_params,
@ -23,10 +17,12 @@ from nilmdb.server.serverutil import (
) )
from nilmdb.utils import serializer_proxy from nilmdb.utils import serializer_proxy
import nilmrun import nilmrun
import nilmrun.processmanager
# Add CORS_allow tool # Add CORS_allow tool
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow) cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
# CherryPy apps # CherryPy apps
class App(object): class App(object):
"""Root application for NILM runner""" """Root application for NILM runner"""
@ -53,6 +49,7 @@ class App(object):
def version(self): def version(self):
return nilmrun.__version__ return nilmrun.__version__
class AppProcess(object): class AppProcess(object):
def __init__(self, manager): def __init__(self, manager):
@ -116,6 +113,7 @@ class AppProcess(object):
self.manager.remove(pid) self.manager.remove(pid)
return status return status
class AppRun(object): class AppRun(object):
def __init__(self, manager): def __init__(self, manager):
self.manager = manager self.manager = manager
@ -154,6 +152,7 @@ class AppRun(object):
"args must be a list of strings") "args must be a list of strings")
return self.manager.run_code(code, args) return self.manager.run_code(code, args)
class Server(object): class Server(object):
def __init__(self, host='127.0.0.1', port=8080, def __init__(self, host='127.0.0.1', port=8080,
force_traceback=False, # include traceback in all errors force_traceback=False, # include traceback in all errors
@ -181,8 +180,10 @@ class Server(object):
# Set up Cross-Origin Resource Sharing (CORS) handler so we # Set up Cross-Origin Resource Sharing (CORS) handler so we
# can correctly respond to browsers' CORS preflight requests. # can correctly respond to browsers' CORS preflight requests.
# This also limits verbs to GET and HEAD by default. # This also limits verbs to GET and HEAD by default.
app_config.update({ 'tools.CORS_allow.on': True, app_config.update({
'tools.CORS_allow.methods': ['GET', 'HEAD'] }) 'tools.CORS_allow.on': True,
'tools.CORS_allow.methods': ['GET', 'HEAD']
})
# Configure the 'json_in' tool to also allow other content-types # Configure the 'json_in' tool to also allow other content-types
# (like x-www-form-urlencoded), and to treat JSON as a dict that # (like x-www-form-urlencoded), and to treat JSON as a dict that
@ -226,10 +227,13 @@ class Server(object):
def stop(self): def stop(self):
cherrypy_stop() cherrypy_stop()
# Multiple processes and threads should be OK here, but we'll still # Multiple processes and threads should be OK here, but we'll still
# follow the NilmDB approach of having just one globally initialized # follow the NilmDB approach of having just one globally initialized
# copy of the server object. # copy of the server object.
_wsgi_server = None _wsgi_server = None
def wsgi_application(basepath): # pragma: no cover def wsgi_application(basepath): # pragma: no cover
"""Return a WSGI application object. """Return a WSGI application object.
@ -247,8 +251,7 @@ def wsgi_application(basepath): # pragma: no cover
except Exception: except Exception:
# Build an error message on failure # Build an error message on failure
import pprint import pprint
err = sprintf("Initializing nilmrun failed:\n\n", err = "Initializing nilmrun failed:\n\n"
dbpath)
err += traceback.format_exc() err += traceback.format_exc()
try: try:
import pwd import pwd
@ -263,8 +266,10 @@ def wsgi_application(basepath): # pragma: no cover
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ)) err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
if _wsgi_server is None: if _wsgi_server is None:
# Serve up the error with our own mini WSGI app. # Serve up the error with our own mini WSGI app.
headers = [ ('Content-type', 'text/plain'), headers = [
('Content-length', str(len(err))) ] ('Content-type', 'text/plain'),
('Content-length', str(len(err)))
]
start_response("500 Internal Server Error", headers) start_response("500 Internal Server Error", headers)
return [err] return [err]

View File

@ -1,2 +1,2 @@
nilmdb>=2.0.2 nilmdb>=2.0.3
psutil>=2.0.0 psutil==5.7.2

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import * from nilmdb.utils.printf import *
@ -13,7 +13,8 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/") def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Kill/remove a process from the NilmRun server', description = 'Kill/remove a process from the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__) version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url', group.add_argument('-u', '--url',

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
import nilmrun.server import nilmrun.server
import argparse import argparse
@ -10,7 +10,8 @@ def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Run the NilmRun server', description = 'Run the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__) version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
@ -34,7 +35,6 @@ def main():
embedded = False embedded = False
server = nilmrun.server.Server(host = args.address, server = nilmrun.server.Server(host = args.address,
port = args.port, port = args.port,
embedded = embedded,
force_traceback = args.traceback) force_traceback = args.traceback)
# Print info # Print info

View File

@ -1,8 +1,8 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import * from nilmdb.utils.printf import *
from nilmdb.utils import datetime_tz import datetime_tz
import nilmrun import nilmrun
import argparse import argparse
@ -13,7 +13,8 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/") def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'List NilmRun processes', description = 'List NilmRun processes',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__) version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url', group.add_argument('-u', '--url',

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import * from nilmdb.utils.printf import *
@ -14,7 +14,8 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/") def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description = 'Run a command on the NilmRun server', description = 'Run a command on the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter, formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__) version=nilmrun.__version__)
group = parser.add_argument_group("Standard options") group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url', group.add_argument('-u', '--url',
@ -30,7 +31,8 @@ def main():
help="Arguments for command") help="Arguments for command")
args = parser.parse_args() args = parser.parse_args()
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify) client = HTTPClient(baseurl=args.url, verify_ssl=not args.noverify,
post_json=True)
# Run command # Run command
pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg }) pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })

View File

@ -28,3 +28,11 @@ versionfile_source=nilmrun/_version.py
versionfile_build=nilmrun/_version.py versionfile_build=nilmrun/_version.py
tag_prefix=nilmrun- tag_prefix=nilmrun-
parentdir_prefix=nilmrun- parentdir_prefix=nilmrun-
[flake8]
exclude=_version.py
extend-ignore=E731
[pylint]
ignore=_version.py
disable=C0103,C0111,R0913,R0914

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
# To release a new version, tag it: # To release a new version, tag it:
# git tag -a nilmrun-1.1 -m "Version 1.1" # git tag -a nilmrun-1.1 -m "Version 1.1"

View File

@ -1,4 +1,4 @@
#!/usr/bin/python #!/usr/bin/env python3
import nose import nose
import os import os

View File

@ -276,11 +276,11 @@ https://creativecommons.org/publicdomain/zero/1.0/ .
""" """
from __future__ import print_function
try: try:
import configparser import configparser
except ImportError: except ImportError:
import configparser as configparser import ConfigParser as configparser
import errno import errno
import json import json
import os import os