Compare commits

..

No commits in common. "master" and "nilmrun-2.0.0" have entirely different histories.

17 changed files with 111 additions and 138 deletions

2
.gitattributes vendored
View File

@ -1 +1 @@
nilmrun/_version.py export-subst
src/_version.py export-subst

2
.gitignore vendored
View File

@ -3,8 +3,8 @@ build/
*.pyc
dist/
nilmrun.egg-info/
.eggs/
# This gets generated as needed by setup.py
MANIFEST.in
MANIFEST

View File

@ -1,8 +0,0 @@
# Root
include README.md
include setup.py
include versioneer.py
include Makefile
# Version
include nilmrun/_version.py

View File

@ -1,6 +1,9 @@
# By default, run the tests.
all: test
test2:
nilmrun/trainola.py data.js
version:
python3 setup.py version
@ -20,11 +23,8 @@ develop:
docs:
make -C docs
ctrl: flake
flake:
flake8 nilmrun
lint:
pylint3 --rcfile=setup.cfg nilmrun
pylint3 --rcfile=.pylintrc nilmdb
test:
ifneq ($(INSIDE_EMACS),)
@ -37,13 +37,12 @@ else
endif
clean::
find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
rm -f .coverage
rm -rf nilmrun.egg-info/ build/
find . -name '*pyc' | xargs rm -f
rm -rf nilmtools.egg-info/ build/ MANIFEST.in
make -C docs clean
gitclean::
git clean -dXf
.PHONY: all version dist sdist install docs test
.PHONY: ctrl lint flake clean gitclean
.PHONY: all version dist sdist install docs lint test clean gitclean

View File

@ -1,3 +1,5 @@
import nilmrun.processmanager
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

View File

@ -41,9 +41,9 @@ def get_config():
cfg = VersioneerConfig()
cfg.VCS = "git"
cfg.style = "pep440"
cfg.tag_prefix = "nilmrun-"
cfg.parentdir_prefix = "nilmrun-"
cfg.versionfile_source = "nilmrun/_version.py"
cfg.tag_prefix = "nilmdb-"
cfg.parentdir_prefix = "nilmdb-"
cfg.versionfile_source = "nilmdb/_version.py"
cfg.verbose = False
return cfg
@ -86,20 +86,20 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
if e.errno == errno.ENOENT:
continue
if verbose:
print("unable to run %s" % dispcmd)
print(("unable to run %s" % dispcmd))
print(e)
return None, None
else:
if verbose:
print("unable to find command, tried %s" % (commands,))
print(("unable to find command, tried %s" % (commands,)))
return None, None
stdout = p.communicate()[0].strip()
if sys.version_info[0] >= 3:
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
print("unable to run %s (error)" % dispcmd)
print("stdout was %s" % stdout)
print(("unable to run %s (error)" % dispcmd))
print(("stdout was %s" % stdout))
return None, p.returncode
return stdout, p.returncode
@ -124,8 +124,8 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
root = os.path.dirname(root) # up a level
if verbose:
print("Tried directories %s but none started with prefix %s" %
(str(rootdirs), parentdir_prefix))
print(("Tried directories %s but none started with prefix %s" %
(str(rootdirs), parentdir_prefix)))
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
@ -192,15 +192,15 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose:
print("discarding '%s', no digits" % ",".join(refs - tags))
print(("discarding '%s', no digits" % ",".join(refs - tags)))
if verbose:
print("likely tags: %s" % ",".join(sorted(tags)))
print(("likely tags: %s" % ",".join(sorted(tags))))
for ref in sorted(tags):
# sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):]
if verbose:
print("picking %s" % r)
print(("picking %s" % r))
return {"version": r,
"full-revisionid": keywords["full"].strip(),
"dirty": False, "error": None,
@ -229,7 +229,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
hide_stderr=True)
if rc != 0:
if verbose:
print("Directory %s not under git control" % root)
print(("Directory %s not under git control" % root))
raise NotThisMethod("'git rev-parse --git-dir' returned error")
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
@ -278,7 +278,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
if not full_tag.startswith(tag_prefix):
if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'"
print(fmt % (full_tag, tag_prefix))
print((fmt % (full_tag, tag_prefix)))
pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
% (full_tag, tag_prefix))
return pieces

View File

@ -1,4 +1,6 @@
#!/usr/bin/env python3
#!/usr/bin/python
from nilmdb.utils.printf import *
import threading
import subprocess
@ -13,18 +15,16 @@ import tempfile
import atexit
import shutil
class ProcessError(Exception):
pass
class LogReceiver(object):
"""Spawn a thread that listens to a pipe for log messages,
and stores them locally."""
def __init__(self, pipe):
self.pipe = pipe
self.log = io.BytesIO()
self.thread = threading.Thread(target=self.run)
self.thread = threading.Thread(target = self.run)
self.thread.start()
def run(self):
@ -41,10 +41,9 @@ class LogReceiver(object):
def clear(self):
self.log = io.BytesIO()
class Process(object):
"""Spawn and manage a subprocess, and capture its output."""
def __init__(self, argv, tempfile=None):
def __init__(self, argv, tempfile = None):
self.start_time = None
# Use a pipe for communicating log data
@ -56,9 +55,9 @@ class Process(object):
# Spawn the new process
try:
self._process = subprocess.Popen(args=argv, stdin=nullfd,
stdout=wpipe, stderr=wpipe,
close_fds=True, cwd="/tmp")
self._process = subprocess.Popen(args = argv, stdin = nullfd,
stdout = wpipe, stderr = wpipe,
close_fds = True, cwd = "/tmp")
except (OSError, TypeError) as e:
raise ProcessError(str(e))
finally:
@ -70,7 +69,7 @@ class Process(object):
self.start_time = time.time()
self.pid = str(uuid.uuid1(self._process.pid or 0))
def _join(self, timeout=1.0):
def _join(self, timeout = 1.0):
start = time.time()
while True:
if self._process.poll() is not None:
@ -79,7 +78,7 @@ class Process(object):
return False
time.sleep(0.1)
def terminate(self, timeout=1.0):
def terminate(self, timeout = 1.0):
"""Terminate a process, and all of its children that are in the same
process group."""
try:
@ -90,19 +89,19 @@ class Process(object):
def getpgid(pid):
try:
return os.getpgid(pid)
except OSError: # pragma: no cover
except OSError: # pragma: no cover
return None
def kill(pid, sig):
try:
return os.kill(pid, sig)
except OSError: # pragma: no cover
except OSError: # pragma: no cover
return
# Find all children
group = getpgid(self._process.pid)
main = psutil.Process(self._process.pid)
allproc = [main] + main.children(recursive=True)
allproc = [ main ] + main.children(recursive = True)
# Kill with SIGTERM, if they're still in this process group
for proc in allproc:
@ -120,7 +119,7 @@ class Process(object):
# See if it worked
return self._join(timeout)
except psutil.Error: # pragma: no cover (race condition)
except psutil.Error: # pragma: no cover (race condition)
return True
def clear_log(self):
@ -143,22 +142,22 @@ class Process(object):
Call .get_info() about a second later."""
try:
main = psutil.Process(self._process.pid)
self._process_list = [main] + main.children(recursive=True)
self._process_list = [ main ] + main.children(recursive = True)
for proc in self._process_list:
proc.cpu_percent(0)
except psutil.Error: # pragma: no cover (race condition)
self._process_list = []
except psutil.Error: # pragma: no cover (race condition)
self._process_list = [ ]
@staticmethod
def get_empty_info():
return {"cpu_percent": 0,
"cpu_user": 0,
"cpu_sys": 0,
"mem_phys": 0,
"mem_virt": 0,
"io_read": 0,
"io_write": 0,
"procs": 0}
return { "cpu_percent": 0,
"cpu_user": 0,
"cpu_sys": 0,
"mem_phys": 0,
"mem_virt": 0,
"io_read": 0,
"io_write": 0,
"procs": 0 }
def get_info(self):
"""Return a dictionary with info about the process CPU and memory
@ -181,7 +180,6 @@ class Process(object):
pass
return d
class ProcessManager(object):
"""Track and manage a collection of Process objects"""
def __init__(self):
@ -193,7 +191,7 @@ class ProcessManager(object):
if pid in self.tmpdirs:
try:
shutil.rmtree(self.tmpdirs[pid])
except OSError: # pragma: no cover
except OSError: # pragma: no cover
pass
del self.tmpdirs[pid]
@ -205,7 +203,7 @@ class ProcessManager(object):
del self.processes[pid]
shutil.rmtree(self.tmpdirs[pid])
del self.tmpdirs[pid]
except Exception: # pragma: no cover
except Exception: # pragma: no cover
pass
def __iter__(self):
@ -220,7 +218,7 @@ class ProcessManager(object):
accessible in the code as sys.argv[1:]."""
# The easiest way to do this, by far, is to just write the
# code to a file. Make a directory to put it in.
tmpdir = tempfile.mkdtemp(prefix="nilmrun-usercode-")
tmpdir = tempfile.mkdtemp(prefix = "nilmrun-usercode-")
try:
# Write the code
codepath = os.path.join(tmpdir, "usercode.py")
@ -231,7 +229,7 @@ class ProcessManager(object):
f.write(repr(args))
# Run the code
argv = [sys.executable, "-B", "-s", "-u", codepath] + args
argv = [ sys.executable, "-B", "-s", "-u", codepath ] + args
pid = self.run_command(argv)
# Save the temp dir
@ -244,7 +242,7 @@ class ProcessManager(object):
if tmpdir is not None:
try:
shutil.rmtree(tmpdir)
except OSError: # pragma: no cover
except OSError: # pragma: no cover
pass
def run_command(self, argv):
@ -262,16 +260,15 @@ class ProcessManager(object):
def get_info(self):
"""Get info about all running PIDs"""
info = {
"total": Process.get_empty_info(),
"pids": {},
"system": {}
}
info = { "total" : Process.get_empty_info(),
"pids" : {},
"system" : {}
}
# Trigger CPU usage collection
for pid in self:
self[pid].get_info_prepare()
psutil.cpu_percent(0, percpu=True)
psutil.cpu_percent(0, percpu = True)
# Give it some time
time.sleep(1)

View File

@ -1,12 +1,18 @@
"""CherryPy-based server for running NILM filters via HTTP"""
import cherrypy
import sys
import os
import socket
import json
import traceback
import time
from nilmdb.utils.printf import sprintf
import nilmdb
from nilmdb.utils.printf import *
from nilmdb.server.serverutil import (
chunked_response,
response_type,
exception_to_httperror,
CORS_allow,
json_to_request_params,
@ -17,12 +23,10 @@ from nilmdb.server.serverutil import (
)
from nilmdb.utils import serializer_proxy
import nilmrun
import nilmrun.processmanager
# Add CORS_allow tool
cherrypy.tools.CORS_allow = cherrypy.Tool('on_start_resource', CORS_allow)
# CherryPy apps
class App(object):
"""Root application for NILM runner"""
@ -49,7 +53,6 @@ class App(object):
def version(self):
return nilmrun.__version__
class AppProcess(object):
def __init__(self, manager):
@ -71,7 +74,7 @@ class AppProcess(object):
# /process/status
@cherrypy.expose
@cherrypy.tools.json_out()
def status(self, pid, clear=False):
def status(self, pid, clear = False):
"""Return status about a process. If clear = True, also clear
the log."""
clear = bool_param(clear)
@ -101,19 +104,18 @@ class AppProcess(object):
@cherrypy.expose
@cherrypy.tools.json_in()
@cherrypy.tools.json_out()
@cherrypy.tools.CORS_allow(methods=["POST"])
@cherrypy.tools.CORS_allow(methods = ["POST"])
def remove(self, pid):
"""Remove a process from the manager, killing it if necessary."""
if pid not in self.manager:
raise cherrypy.HTTPError("404 Not Found", "No such PID")
if not self.manager.terminate(pid): # pragma: no cover
if not self.manager.terminate(pid): # pragma: no cover
raise cherrypy.HTTPError("503 Service Unavailable",
"Failed to stop process")
status = self.process_status(pid)
self.manager.remove(pid)
return status
class AppRun(object):
def __init__(self, manager):
self.manager = manager
@ -123,7 +125,7 @@ class AppRun(object):
@cherrypy.tools.json_in()
@cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods=["POST"])
@cherrypy.tools.CORS_allow(methods = ["POST"])
def command(self, argv):
"""Execute an arbitrary program on the server. argv is a
list of the program and its arguments: 'argv[0]' is the program
@ -138,8 +140,8 @@ class AppRun(object):
@cherrypy.tools.json_in()
@cherrypy.tools.json_out()
@exception_to_httperror(nilmrun.processmanager.ProcessError)
@cherrypy.tools.CORS_allow(methods=["POST"])
def code(self, code, args=None):
@cherrypy.tools.CORS_allow(methods = ["POST"])
def code(self, code, args = None):
"""Execute arbitrary Python code. 'code' is a formatted string.
It will be run as if it were written into a Python file and
executed. 'args' is a list of strings, and they are passed
@ -152,11 +154,10 @@ class AppRun(object):
"args must be a list of strings")
return self.manager.run_code(code, args)
class Server(object):
def __init__(self, host='127.0.0.1', port=8080,
force_traceback=False, # include traceback in all errors
basepath='', # base URL path for cherrypy.tree
def __init__(self, host = '127.0.0.1', port = 8080,
force_traceback = False, # include traceback in all errors
basepath = '', # base URL path for cherrypy.tree
):
# Build up global server configuration
@ -175,25 +176,23 @@ class Server(object):
})
# Some default headers to just help identify that things are working
app_config.update({'response.headers.X-Jim-Is-Awesome': 'yeah'})
app_config.update({ 'response.headers.X-Jim-Is-Awesome': 'yeah' })
# Set up Cross-Origin Resource Sharing (CORS) handler so we
# can correctly respond to browsers' CORS preflight requests.
# This also limits verbs to GET and HEAD by default.
app_config.update({
'tools.CORS_allow.on': True,
'tools.CORS_allow.methods': ['GET', 'HEAD']
})
app_config.update({ 'tools.CORS_allow.on': True,
'tools.CORS_allow.methods': ['GET', 'HEAD'] })
# Configure the 'json_in' tool to also allow other content-types
# (like x-www-form-urlencoded), and to treat JSON as a dict that
# fills requests.param.
app_config.update({'tools.json_in.force': False,
'tools.json_in.processor': json_to_request_params})
app_config.update({ 'tools.json_in.force': False,
'tools.json_in.processor': json_to_request_params })
# Send tracebacks in error responses. They're hidden by the
# error_page function for client errors (code 400-499).
app_config.update({'request.show_tracebacks': True})
app_config.update({ 'request.show_tracebacks' : True })
self.force_traceback = force_traceback
# Patch CherryPy error handler to never pad out error messages.
@ -211,7 +210,7 @@ class Server(object):
root.process = AppProcess(manager)
root.run = AppRun(manager)
cherrypy.tree.apps = {}
cherrypy.tree.mount(root, basepath, config={"/": app_config})
cherrypy.tree.mount(root, basepath, config = { "/" : app_config })
# Set up the WSGI application pointer for external programs
self.wsgi_application = cherrypy.tree
@ -221,20 +220,17 @@ class Server(object):
return json_error_page(status, message, traceback, version,
self.force_traceback)
def start(self, blocking=False, event=None):
def start(self, blocking = False, event = None):
cherrypy_start(blocking, event)
def stop(self):
cherrypy_stop()
# Multiple processes and threads should be OK here, but we'll still
# follow the NilmDB approach of having just one globally initialized
# copy of the server object.
_wsgi_server = None
def wsgi_application(basepath): # pragma: no cover
def wsgi_application(basepath): # pragma: no cover
"""Return a WSGI application object.
'basepath' is the URL path of the application base, which
@ -247,11 +243,12 @@ def wsgi_application(basepath): # pragma: no cover
# Try to start the server
try:
_wsgi_server = nilmrun.server.Server(
basepath=basepath.rstrip('/'))
basepath = basepath.rstrip('/'))
except Exception:
# Build an error message on failure
import pprint
err = "Initializing nilmrun failed:\n\n"
err = sprintf("Initializing nilmrun failed:\n\n",
dbpath)
err += traceback.format_exc()
try:
import pwd
@ -266,10 +263,8 @@ def wsgi_application(basepath): # pragma: no cover
err += sprintf("\nEnvironment:\n%s\n", pprint.pformat(environ))
if _wsgi_server is None:
# Serve up the error with our own mini WSGI app.
headers = [
('Content-type', 'text/plain'),
('Content-length', str(len(err)))
]
headers = [ ('Content-type', 'text/plain'),
('Content-length', str(len(err))) ]
start_response("500 Internal Server Error", headers)
return [err]

View File

@ -1,2 +1,2 @@
nilmdb>=2.0.3
psutil==5.7.2
nilmdb>=2.0.2
psutil>=2.0.0

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
@ -13,9 +13,8 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'Kill/remove a process from the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
version = nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
import nilmrun.server
import argparse
@ -10,9 +10,8 @@ def main():
parser = argparse.ArgumentParser(
description = 'Run the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
version = nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-a', '--address',
@ -35,6 +34,7 @@ def main():
embedded = False
server = nilmrun.server.Server(host = args.address,
port = args.port,
embedded = embedded,
force_traceback = args.traceback)
# Print info

View File

@ -1,8 +1,8 @@
#!/usr/bin/env python3
#!/usr/bin/python
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
import datetime_tz
from nilmdb.utils import datetime_tz
import nilmrun
import argparse
@ -13,9 +13,8 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'List NilmRun processes',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
version = nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
from nilmdb.client.httpclient import HTTPClient, ClientError, ServerError
from nilmdb.utils.printf import *
@ -14,9 +14,8 @@ def main():
def_url = os.environ.get("NILMRUN_URL", "http://localhost/nilmrun/")
parser = argparse.ArgumentParser(
description = 'Run a command on the NilmRun server',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-v", "--version", action="version",
version=nilmrun.__version__)
formatter_class = argparse.ArgumentDefaultsHelpFormatter,
version = nilmrun.__version__)
group = parser.add_argument_group("Standard options")
group.add_argument('-u', '--url',
help = 'NilmRun server URL', default = def_url)
@ -31,8 +30,7 @@ def main():
help="Arguments for command")
args = parser.parse_args()
client = HTTPClient(baseurl=args.url, verify_ssl=not args.noverify,
post_json=True)
client = HTTPClient(baseurl = args.url, verify_ssl = not args.noverify)
# Run command
pid = client.post("run/command", { "argv": [ args.cmd ] + args.arg })

View File

@ -28,11 +28,3 @@ versionfile_source=nilmrun/_version.py
versionfile_build=nilmrun/_version.py
tag_prefix=nilmrun-
parentdir_prefix=nilmrun-
[flake8]
exclude=_version.py
extend-ignore=E731
[pylint]
ignore=_version.py
disable=C0103,C0111,R0913,R0914

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
# To release a new version, tag it:
# git tag -a nilmrun-1.1 -m "Version 1.1"

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python
import nose
import os

View File

@ -276,11 +276,11 @@ https://creativecommons.org/publicdomain/zero/1.0/ .
"""
from __future__ import print_function
try:
import configparser
except ImportError:
import ConfigParser as configparser
import configparser as configparser
import errno
import json
import os