Browse Source

Improve pipewatch; add nilm-pipewatch script

tags/nilmtools-1.4.0
Jim Paris 10 years ago
parent
commit
def465b57c
3 changed files with 69 additions and 40 deletions
  1. +4
    -1
      Makefile
  2. +64
    -39
      nilmtools/pipewatch.py
  3. +1
    -0
      setup.py

+ 4
- 1
Makefile View File

@@ -8,7 +8,10 @@ else
@echo "Try 'make install'" @echo "Try 'make install'"
endif endif


test: test_trainola
test: test_pipewatch

test_pipewatch:
nilmtools/pipewatch.py -t 3 "seq 10 20" "seq 20 30"


test_trainola: test_trainola:
-nilmtool -u http://bucket/nilmdb remove -s min -e max \ -nilmtool -u http://bucket/nilmdb remove -s min -e max \


+ 64
- 39
nilmtools/pipewatch.py View File

@@ -14,6 +14,7 @@ import tempfile
import threading import threading
import select import select
import signal import signal
import Queue


def parse_args(argv = None): def parse_args(argv = None):
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
@@ -45,8 +46,24 @@ def parse_args(argv = None):


return args return args


def sigpipe_fixup():
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
def reader_thread(queue, fd):
# Read from a file descriptor, write to queue.
try:
while True:
(r, w, x) = select.select([fd], [], [fd], 0.25)
if x:
raise Exception
if not r:
# short timeout -- just try again. This is to catch the
# fd being closed elsewhere, which is only detected
# when select restarts.
continue
data = os.read(fd, 65536)
if data == "":
raise Exception
queue.put(data)
except Exception:
queue.put(None)


def main(argv = None): def main(argv = None):
args = parse_args(argv) args = parse_args(argv)
@@ -59,49 +76,57 @@ def main(argv = None):




with open(os.devnull, "r") as devnull: with open(os.devnull, "r") as devnull:
sigpipe_fixup()
generator = subprocess.Popen(args.generator, shell = True, generator = subprocess.Popen(args.generator, shell = True,
bufsize = -1, close_fds = True, bufsize = -1, close_fds = True,
stdin = devnull, stdin = devnull,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = None,
preexec_fn = sigpipe_fixup)
stderr = None)
consumer = subprocess.Popen(args.consumer, shell = True, consumer = subprocess.Popen(args.consumer, shell = True,
bufsize = -1, close_fds = True,
bufsize = -11, close_fds = True,
stdin = subprocess.PIPE, stdin = subprocess.PIPE,
stdout = None, stderr = None,
preexec_fn = sigpipe_fixup)

stdout = generator.stdout.fileno()
stdin = consumer.stdin.fileno()
while True:
( r, w, x ) = select.select([stdout], [], [stdout,stdin],
args.timeout)
if not r and not x:
fprintf(sys.stderr, "pipewatch: generator timeout\n")
break
if x:
fprintf(sys.stderr, "pipewatch: I/O error\n")
break
print r, w, x
print "reading"
data = os.read(stdout, 65536)
if data == "":
fprintf(sys.stderr, "generator EOF\n")
break
os.write(stdin, data)

generator.stdout.close()
consumer.stdin.close()

def wait_kill(proc):
start = time.time()

gret = generator.wait()
cret = consumer.wait()

fprintf(sys.stderr, "generator returned %d, " +
stdout = None, stderr = None)

queue = Queue.Queue(maxsize = 32)
reader = threading.Thread(target = reader_thread,
args = (queue, generator.stdout.fileno()))
reader.start()
try:
while True:
try:
data = queue.get(True, args.timeout)
if data is None:
break
consumer.stdin.write(data)
except Queue.Empty:
# Timeout: kill the generator
fprintf(sys.stderr, "pipewatch: timeout\n")
generator.terminate()
break

generator.stdout.close()
consumer.stdin.close()
except IOError:
fprintf(sys.stderr, "pipewatch: I/O error\n")

def kill(proc):
# Wait for a process to end, or kill it
def poll_timeout(proc, timeout):
for x in range(1+int(timeout / 0.1)):
if proc.poll() is not None:
break
time.sleep(0.1)
return proc.poll()
if poll_timeout(proc, 0.5) is None:
proc.terminate()
if poll_timeout(proc, 0.5) is None:
proc.kill()
return poll_timeout(proc, 0.5)

# Wait for them to die, or kill them
gret = kill(generator)
cret = kill(consumer)

fprintf(sys.stderr, "pipewatch: generator returned %d, " +
"consumer returned %d\n", gret, cret) "consumer returned %d\n", gret, cret)
if gret == 0 and cret == 0: if gret == 0 and cret == 0:
sys.exit(0) sys.exit(0)


+ 1
- 0
setup.py View File

@@ -80,6 +80,7 @@ setup(name='nilmtools',
'nilm-cleanup = nilmtools.cleanup:main', 'nilm-cleanup = nilmtools.cleanup:main',
'nilm-median = nilmtools.median:main', 'nilm-median = nilmtools.median:main',
'nilm-trainola = nilmtools.trainola:main', 'nilm-trainola = nilmtools.trainola:main',
'nilm-pipewatch = nilmtools.pipewatch:main',
], ],
}, },
zip_safe = False, zip_safe = False,


Loading…
Cancel
Save