Freeze requirements to specific versions

This was never updated when timestamps were changed from floats to
ints.

README.md

@@ -6,14 +6,13 @@ NilmDB requires Python 3.7 or newer.
 ## Prerequisites:
 
     # Runtime and build environments
-    sudo apt install python3.7 python3.7-dev python3.7-venv python3-pip
+    sudo apt install python3 python3-dev python3-venv python3-pip
 
-    # Optional: create a new Python virtual environment to isolate
-    # dependencies.  To leave the virtual environment, run "deactivate"
-    python -m venv venv
-    source venv/bin/activate
+    # Create a new Python virtual environment to isolate deps.
+    python3 -m venv ../venv
+    source ../venv/bin/activate   # run "deactivate" to leave
 
-    # Install all Python dependencies from PyPI.
+    # Install all Python dependencies
     pip3 install -r requirements.txt
 
 ## Test:
@@ -22,6 +21,14 @@ NilmDB requires Python 3.7 or newer.
 
 ## Install:
 
+Install it into the virtual environment
+
+    python3 setup.py install
+
+If you want to instead install it system-wide, you will also need to
+install the requirements system-wide:
+
+    sudo pip3 install -r requirements.txt
     sudo python3 setup.py install
 
 ## Usage:

extras/fix-oversize-files.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import os
 import sys

nilmdb/client/client.py

@@ -145,8 +145,8 @@ class Client():
 
         Example:
           with client.stream_insert_context('/path', start, end) as ctx:
-            ctx.insert('1234567890.0 1 2 3 4\\n')
-            ctx.insert('1234567891.0 1 2 3 4\\n')
+            ctx.insert('1234567890000000 1 2 3 4\\n')
+            ctx.insert('1234567891000000 1 2 3 4\\n')
 
         For more details, see help for nilmdb.client.client.StreamInserter
 

nilmdb/client/httpclient.py

@@ -109,7 +109,7 @@ class HTTPClient():
                                           stream=False, headers=headers)
         if isjson:
             return json.loads(response.content)
-        return response.content
+        return response.text
 
     def get(self, url, params=None):
         """Simple GET (parameters in URL)"""

nilmdb/cmdline/cmdline.py

@@ -66,7 +66,7 @@ class Complete():
         layouts = []
         for i in range(1, 10):
             layouts.extend([(t + "_" + str(i)) for t in types])
-        return (l for l in layouts if l.startswith(prefix))
+        return (lay for lay in layouts if lay.startswith(prefix))
 
     def meta_key(self, prefix, parsed_args, **kwargs):
         return (kv.split('=')[0] for kv

nilmdb/scripts/nilmdb_fsck.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import nilmdb.fsck
 import argparse

nilmdb/scripts/nilmdb_server.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import os
 import sys

nilmdb/scripts/nilmtool.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import nilmdb.cmdline
 

nilmdb/server/rocket.c

@@ -168,7 +168,7 @@ static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
 	if (!layout)
 		return -1;
 	if (path) {
-                if (strlen(path) != pathlen) {
+                if (strlen(path) != (size_t)pathlen) {
                         PyErr_SetString(PyExc_ValueError, "path must not "
                                         "contain NUL characters");
                         return -1;
@@ -477,7 +477,7 @@ static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
         }
 
         /* Write binary data */
-        if (fwrite(data, self->binary_size, rows, self->file) != rows) {
+        if (fwrite(data, self->binary_size, rows, self->file) != (size_t)rows) {
                 PyErr_SetFromErrno(PyExc_OSError);
                 return NULL;
         }
@@ -628,7 +628,7 @@ static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args)
 
         /* Data in the file is already in the desired little-endian
            binary format, so just read it directly. */
-        if (fread(str, self->binary_size, count, self->file) != count) {
+        if (fread(str, self->binary_size, count, self->file) != (size_t)count) {
                 free(str);
                 PyErr_SetFromErrno(PyExc_OSError);
                 return NULL;

nilmdb/server/serverutil.py

@@ -6,6 +6,7 @@ import sys
 import json
 import decorator
 import functools
+import threading
 
 import cherrypy
 
@@ -178,6 +179,19 @@ def cherrypy_patch_exit():
             os._exit = real_exit
     bus.exit = functools.partial(patched_exit, bus.exit)
 
+    # A behavior change in Python 3.8 means that some thread exceptions,
+    # derived from SystemExit, now print tracebacks where they didn't
+    # used to: https://bugs.python.org/issue1230540
+    # Install a thread exception hook that ignores CherryPyExit;
+    # to make this match the behavior where we didn't set
+    # threading.excepthook, we also need to ignore SystemExit.
+    def hook(args):
+        if args.exc_type == CherryPyExit or args.exc_type == SystemExit:
+            return
+        sys.excepthook(args.exc_type, args.exc_value,
+                       args.exc_traceback)  # pragma: no cover
+    threading.excepthook = hook
+
 
 # Start/stop CherryPy standalone server
 def cherrypy_start(blocking=False, event=False):

nilmdb/utils/timestamper.py

@@ -26,7 +26,9 @@ class Timestamper():
                 return b""
             if line[0:1] == b'#':
                 continue
-            break
+            # For some reason, coverage on python 3.8 reports that
+            # we never hit this break, even though we definitely do.
+            break  # pragma: no cover
         try:
             return next(self.ts_iter) + line
         except StopIteration:

requirements.txt

@@ -1,16 +1,40 @@
-argcomplete>=1.10.0
-CherryPy>=18.1.2
-coverage>=4.5.4
-cython>=0.29.13
-decorator>=4.4.0
-fallocate>=1.6.4
-flake8>=3.7.8
-nose>=1.3.7
-numpy>=1.17.0
-progressbar>=2.5
-psutil>=5.6.3
-python-datetime-tz>=0.5.4
-python-dateutil>=2.8.0
-requests>=2.22.0
-tz>=0.2.2
-WebTest>=2.0.33
+argcomplete==1.12.0
+CherryPy==18.6.0
+coverage==5.2.1
+Cython==0.29.21
+decorator==4.4.2
+fallocate==1.6.4
+flake8==3.8.3
+nose==1.3.7
+numpy==1.19.1
+progressbar==2.5
+psutil==5.7.2
+python-datetime-tz==0.5.4
+python-dateutil==2.8.1
+requests==2.24.0
+tz==0.2.2
+WebTest==2.0.35
+
+## The following requirements were added by pip freeze:
+beautifulsoup4==4.9.1
+certifi==2020.6.20
+chardet==3.0.4
+cheroot==8.4.2
+idna==2.10
+jaraco.classes==3.1.0
+jaraco.collections==3.0.0
+jaraco.functools==3.0.1
+jaraco.text==3.2.0
+mccabe==0.6.1
+more-itertools==8.4.0
+portend==2.6
+pycodestyle==2.6.0
+pyflakes==2.2.0
+pytz==2020.1
+six==1.15.0
+soupsieve==2.0.1
+tempora==4.0.0
+urllib3==1.25.10
+waitress==1.4.4
+WebOb==1.8.6
+zc.lockfile==2.0

setup.py

@@ -1,10 +1,10 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 # To release a new version, tag it:
 #   git tag -a nilmdb-1.1 -m "Version 1.1"
 #   git push --tags
 # Then just package it up:
-#   python setup.py sdist
+#   python3 setup.py sdist
 
 import sys
 import os
@@ -36,7 +36,7 @@ install_requires = open('requirements.txt').readlines()
 setup(name='nilmdb',
       version = versioneer.get_version(),
       cmdclass = versioneer.get_cmdclass(),
-      url = 'https://git.jim.sh/jim/lees/nilmdb.git',
+      url = 'https://git.jim.sh/nilm/nilmdb.git',
       author = 'Jim Paris',
       description = "NILM Database",
       long_description = "NILM Database",

tests/runtests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import nose
 import os