Merge branch 'py3'

.coveragerc

@@ -1,10 +1,11 @@
 # -*- conf -*-
 
 [run]
-# branch = True
+branch = True
 
 [report]
 exclude_lines =
 	pragma: no cover
 	if 0:
-omit = nilmdb/utils/datetime_tz*,nilmdb/scripts,nilmdb/_version.py,nilmdb/fsck
+omit = nilmdb/scripts,nilmdb/_version.py,nilmdb/fsck
+show_missing = True

.gitignore

@@ -15,10 +15,7 @@ nilmdb/server/rbtree.c
 # Setup junk
 dist/
 nilmdb.egg-info/
-
-# This gets generated as needed by setup.py
-MANIFEST.in
-MANIFEST
+venv/
 
 # Misc
 timeit*out

MANIFEST.in

@@ -0,0 +1,29 @@
+# Root
+include README.txt
+include setup.cfg
+include setup.py
+include versioneer.py
+include Makefile
+include .coveragerc
+include .pylintrc
+include requirements.txt
+
+# Cython files -- include .pyx source, but not the generated .c files
+# (Downstream systems must have cython installed in order to build)
+recursive-include nilmdb/server *.pyx *.pyxdep *.pxd
+exclude nilmdb/server/interval.c
+exclude nilmdb/server/rbtree.c
+
+# Version
+include nilmdb/_version.py
+
+# Tests
+recursive-include tests *.py
+recursive-include tests/data *
+include tests/test.order
+
+# Docs
+recursive-include docs Makefile *.md
+
+# Extras
+recursive-include extras *

Makefile

@@ -2,42 +2,42 @@
 all: test
 
 version:
-	python setup.py version
+	python3 setup.py version
 
 build:
-	python setup.py build_ext --inplace
+	python3 setup.py build_ext --inplace
 
 dist: sdist
 sdist:
-	python setup.py sdist
+	python3 setup.py sdist
 
 install:
-	python setup.py install
+	python3 setup.py install
 
 develop:
-	python setup.py develop
+	python3 setup.py develop
 
 docs:
 	make -C docs
 
 lint:
-	pylint --rcfile=.pylintrc nilmdb
+	pylint3 --rcfile=.pylintrc nilmdb
 
 test:
-ifeq ($(INSIDE_EMACS), t)
+ifneq ($(INSIDE_EMACS),)
 # Use the slightly more flexible script
-	python setup.py build_ext --inplace
-	python tests/runtests.py
+	python3 setup.py build_ext --inplace
+	python3 tests/runtests.py
 else
 # Let setup.py check dependencies, build stuff, and run the test
-	python setup.py nosetests
+	python3 setup.py nosetests
 endif
 
 clean::
-	find . -name '*pyc' | xargs rm -f
+	find . -name '*.pyc' -o -name '__pycache__' -print0 | xargs -0 rm -rf
 	rm -f .coverage
 	rm -rf tests/*testdb*
-	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so MANIFEST.in
+	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so
 	make -C docs clean
 
 gitclean::

README.txt

@@ -4,25 +4,23 @@ by Jim Paris <jim@jtan.com>
 Prerequisites:
 
   # Runtime and build environments
-  sudo apt-get install python2.7 python2.7-dev python-setuptools cython
+  sudo apt install python3.7 python3.7-dev python3.7-venv python3-pip
 
-  # Base NilmDB dependencies
-  sudo apt-get install python-cherrypy3 python-decorator python-simplejson
-  sudo apt-get install python-requests python-dateutil python-tz
-  sudo apt-get install python-progressbar python-psutil
+  # Optional: create a new Python virtual environment to isolate
+  # dependencies.  To leave the virtual environment, run "deactivate"
+  python -m venv venv
+  source venv/bin/activate
 
-  # Other dependencies (required by some modules)
-  sudo apt-get install python-numpy
-
-  # Tools for running tests
-  sudo apt-get install python-nose python-coverage
+  # Install all Python dependencies from PyPI.
+  pip3 install -r requirements.txt
 
 Test:
-  python setup.py nosetests
+
+  python3 setup.py nosetests
 
 Install:
 
-  python setup.py install
+  sudo python3 setup.py install
 
 Usage:
 

docs/design.md

@@ -430,7 +430,7 @@ mod_wsgi requires "WSGIChunkedRequest On" to handle
 "Transfer-encoding: Chunked" requests.  However, `/stream/insert`
 doesn't handle this correctly right now, because:
 
-- The `cherrpy.request.body.read()` call needs to be fixed for chunked requests
+- The `cherrypy.request.body.read()` call needs to be fixed for chunked requests
 
 - We don't want to just buffer endlessly in the server, and it will
   require some thought on how to handle data in chunks (what to do about
@@ -438,3 +438,32 @@ doesn't handle this correctly right now, because:
 
 It is probably better to just keep the endpoint management on the client
 side, so leave "WSGIChunkedRequest off" for now.
+
+
+Unicode & character encoding
+----------------------------
+
+Stream data is passed back and forth as raw `bytes` objects in most
+places, including the `nilmdb.client` and command-line interfaces.
+This is done partially for performance reasons, and partially to
+support the binary insert/extract options, where character-set encoding
+would not apply.
+
+For the HTTP server, the raw bytes transferred over HTTP are interpreted
+as follows:
+- For `/stream/insert`, the client-provided `Content-Type` is ignored,
+  and the data is read as if it were `application/octet-stream`.
+- For `/stream/extract`, the returned data is `application/octet-stream`.
+- All other endpoints communicate via JSON, which is specified to always
+  be encoded as UTF-8.  This includes:
+    - `/version`
+    - `/dbinfo`
+    - `/stream/list`
+    - `/stream/create`
+    - `/stream/destroy`
+    - `/stream/rename`
+    - `/stream/get_metadata`
+    - `/stream/set_metadata`
+    - `/stream/update_metadata`
+    - `/stream/remove`
+    - `/stream/intervals`

extras/fix-oversize-files.py

@@ -2,7 +2,7 @@
 
 import os
 import sys
-import cPickle as pickle
+import pickle
 import argparse
 import fcntl
 import re
@@ -44,7 +44,7 @@ with open(lock, "w") as f:
                 maxsize = fix[fixpath]
                 if size > maxsize:
                     diff = size - maxsize
-                    print diff, "too big:", fn
+                    print(diff, "too big:", fn)
                     if args.yes:
                         with open(fn, "a+") as dbfile:
                             dbfile.truncate(maxsize)

nilmdb/__init__.py

@@ -8,3 +8,7 @@
 from nilmdb._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions

nilmdb/_version.py

@@ -1,197 +1,520 @@
 
-IN_LONG_VERSION_PY = True
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
-
-# these strings will be replaced by git during git-archive
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
+# versioneer-0.18 (https://github.com/warner/python-versioneer)
 
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.cmd on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    git_date = "$Format:%ci$"
+    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = "nilmdb-"
+    cfg.parentdir_prefix = "nilmdb-"
+    cfg.versionfile_source = "nilmdb/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+                env=None):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+                                 stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None, None
+    else:
         if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
-        return None
+            print("unable to find command, tried %s" % (commands,))
+        return None, None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %s (error)" % args[0])
-        return None
-    return stdout
+            print("unable to run %s (error)" % dispcmd)
+            print("stdout was %s" % stdout)
+        return None, p.returncode
+    return stdout, p.returncode
 
 
-import sys
-import re
-import os.path
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes both
+    the project name and a version string. We will also support searching up
+    two directory levels for an appropriately named parent directory
+    """
+    rootdirs = []
+
+    for i in range(3):
+        dirname = os.path.basename(root)
+        if dirname.startswith(parentdir_prefix):
+            return {"version": dirname[len(parentdir_prefix):],
+                    "full-revisionid": None,
+                    "dirty": False, "error": None, "date": None}
+        else:
+            rootdirs.append(root)
+            root = os.path.dirname(root)  # up a level
+
+    if verbose:
+        print("Tried directories %s but none started with prefix %s" %
+              (str(rootdirs), parentdir_prefix))
+    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+
 
-def get_expanded_variables(versionfile_source):
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        for line in open(versionfile_source,"r").readlines():
+        f = open(versionfile_abs, "r")
+        for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
+            if line.strip().startswith("git_date ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["date"] = mo.group(1)
+        f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    date = keywords.get("date")
+    if date is not None:
+        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+        # -like" string, which we must then edit to make compliant), because
+        # it's been around since git-1.5.3, and it's too difficult to
+        # discover which version we're using, or to work around using an
+        # older one.
+        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
-    for ref in list(refs):
-        if not re.search(r'\d', ref):
-            if verbose:
-                print("discarding '%s', no digits" % ref)
-            refs.discard(ref)
-            # Assume all version tags have a digit. git's %d expansion
-            # behaves like git log --decorate=short and strips out the
-            # refs/heads/ and refs/tags/ prefixes that would let us
-            # distinguish between branches and tags. By ignoring refnames
-            # without digits, we filter out many common branch names like
-            # "release" and "stabilization", as well as "HEAD" and "master".
+    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
+    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
+    TAG = "tag: "
+    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    if not tags:
+        # Either we're using git < 1.8.3, or there really are no tags. We use
+        # a heuristic: assume all version tags have a digit. The old git %d
+        # expansion behaves like git log --decorate=short and strips out the
+        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
+        # between branches and tags. By ignoring refnames without digits, we
+        # filter out many common branch names like "release" and
+        # "stabilization", as well as "HEAD" and "master".
+        tags = set([r for r in refs if re.search(r'\d', r)])
+        if verbose:
+            print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
-        print("remaining refs: %s" % ",".join(sorted(refs)))
-    for ref in sorted(refs):
+        print("likely tags: %s" % ",".join(sorted(tags)))
+    for ref in sorted(tags):
         # sorting will prefer e.g. "2.0" over "2.0rc1"
         if ref.startswith(tag_prefix):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %s" % r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None,
+                    "date": date}
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags", "date": None}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
-    if not os.path.exists(os.path.join(root, ".git")):
-        if verbose:
-            print("no .git in %s" % root)
-        return {}
 
-    GIT = "git"
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
+        GITS = ["git.cmd", "git.exe"]
+
+    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+                          hide_stderr=True)
+    if rc != 0:
         if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
+            print("Directory %s not under git control" % root)
+        raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+                                          "--always", "--long",
+                                          "--match", "%s*" % tag_prefix],
+                                   cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                    cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    # commit date: see ISO-8601 comment in git_versions_from_keywords()
+    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
+                       cwd=root)[0].strip()
+    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"],
+                "date": None}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None,
+            "date": pieces.get("date")}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
         # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
-    else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
-
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = "nilmdb-"
-parentdir_prefix = "nilmdb-"
-versionfile_source = "nilmdb/_version.py"
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = { "refnames": git_refnames, "full": git_full }
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree",
+                "date": None}
+
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
 
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version", "date": None}

nilmdb/client/client.py

@@ -7,7 +7,7 @@ import nilmdb.client.httpclient
 from nilmdb.client.errors import ClientError
 
 import time
-import simplejson as json
+import json
 import contextlib
 
 from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
@@ -159,7 +159,7 @@ class Client(object):
         so it will be broken into reasonably-sized chunks and
         start/end will be deduced if missing."""
         with self.stream_insert_context(path, start, end) as ctx:
-            if isinstance(data, basestring):
+            if isinstance(data, bytes):
                 ctx.insert(data)
             else:
                 for chunk in data:
@@ -181,7 +181,7 @@ class Client(object):
         }
         if binary:
             params["binary"] = 1
-        return self.http.put("stream/insert", data, params, binary = binary)
+        return self.http.put("stream/insert", data, params)
 
     def stream_intervals(self, path, start = None, end = None, diffpath = None):
         """
@@ -331,7 +331,7 @@ class StreamInserter(object):
         # Send the block once we have enough data
         if self._block_len >= maxdata:
             self._send_block(final = False)
-            if self._block_len >= self._max_data_after_send: # pragma: no cover
+            if self._block_len >= self._max_data_after_send:
                 raise ValueError("too much data left over after trying"
                                  " to send intermediate block; is it"
                                  " missing newlines or malformed?")
@@ -370,10 +370,10 @@ class StreamInserter(object):
         there isn't one."""
         start = 0
         while True:
-            end = block.find('\n', start)
+            end = block.find(b'\n', start)
             if end < 0:
                 raise IndexError
-            if block[start] != '#':
+            if block[start] != b'#'[0]:
                 return (start, (end + 1))
             start = end + 1
 
@@ -381,12 +381,12 @@ class StreamInserter(object):
         """Return the (start, end) indices of the last full line in
         block[:length] that isn't a comment, or raise IndexError if
         there isn't one."""
-        end = block.rfind('\n')
+        end = block.rfind(b'\n')
         if end <= 0:
             raise IndexError
         while True:
-            start = block.rfind('\n', 0, end)
-            if block[start + 1] != '#':
+            start = block.rfind(b'\n', 0, end)
+            if block[start + 1] != b'#'[0]:
                 return ((start + 1), end)
             if start == -1:
                 raise IndexError
@@ -396,7 +396,7 @@ class StreamInserter(object):
         """Send data currently in the block.  The data sent will
         consist of full lines only, so some might be left over."""
         # Build the full string to send
-        block = "".join(self._block_data)
+        block = b"".join(self._block_data)
 
         start_ts = self._interval_start
         if start_ts is None:
@@ -413,7 +413,7 @@ class StreamInserter(object):
             # or the timestamp of the last line plus epsilon.
             end_ts = self._interval_end
             try:
-                if block[-1] != '\n':
+                if block[-1] != b'\n'[0]:
                     raise ValueError("final block didn't end with a newline")
                 if end_ts is None:
                     (spos, epos) = self._get_last_noncomment(block)

nilmdb/client/errors.py

@@ -9,7 +9,7 @@ class Error(Exception):
                  message = None,
                  url = None,
                  traceback = None):
-        Exception.__init__(self, status)
+        super().__init__(status)
         self.status = status     # e.g. "400 Bad Request"
         self.message = message   # textual message from the server
         self.url = url           # URL we were requesting
@@ -18,14 +18,14 @@ class Error(Exception):
         s = sprintf("[%s]", self.status)
         if self.message:
             s += sprintf(" %s", self.message)
-        if show_url and self.url: # pragma: no cover
+        if show_url and self.url:
             s += sprintf(" (%s)", self.url)
-        if self.traceback: # pragma: no cover
+        if self.traceback:
             s += sprintf("\nServer traceback:\n%s", self.traceback)
         return s
     def __str__(self):
         return self._format_error(show_url = False)
-    def __repr__(self): # pragma: no cover
+    def __repr__(self):
         return self._format_error(show_url = True)
 class ClientError(Error):
     pass

nilmdb/client/httpclient.py

@@ -3,8 +3,8 @@
 import nilmdb.utils
 from nilmdb.client.errors import ClientError, ServerError, Error
 
-import simplejson as json
-import urlparse
+import json
+import urllib.parse
 import requests
 
 class HTTPClient(object):
@@ -13,9 +13,9 @@ class HTTPClient(object):
         """If baseurl is supplied, all other functions that take
         a URL can be given a relative URL instead."""
         # Verify / clean up URL
-        reparsed = urlparse.urlparse(baseurl).geturl()
+        reparsed = urllib.parse.urlparse(baseurl).geturl()
         if '://' not in reparsed:
-            reparsed = urlparse.urlparse("http://" + baseurl).geturl()
+            reparsed = urllib.parse.urlparse("http://" + baseurl).geturl()
         self.baseurl = reparsed.rstrip('/') + '/'
 
         # Note whether we want SSL verification
@@ -42,11 +42,11 @@ class HTTPClient(object):
             args["status"] = jsonerror["status"]
             args["message"] = jsonerror["message"]
             args["traceback"] = jsonerror["traceback"]
-        except Exception: # pragma: no cover
+        except Exception:
             pass
         if code >= 400 and code <= 499:
             raise ClientError(**args)
-        else: # pragma: no cover
+        else:
             if code >= 500 and code <= 599:
                 if args["message"] is None:
                     args["message"] = ("(no message; try disabling " +
@@ -60,7 +60,7 @@ class HTTPClient(object):
         pass
 
     def _do_req(self, method, url, query_data, body_data, stream, headers):
-        url = urlparse.urljoin(self.baseurl, url)
+        url = urllib.parse.urljoin(self.baseurl, url)
         try:
             # Create a new session, ensure we send "Connection: close",
             # and explicitly close connection after the transfer.
@@ -87,7 +87,7 @@ class HTTPClient(object):
             session.close()
         except requests.RequestException as e:
             raise ServerError(status = "502 Error", url = url,
-                              message = str(e.message))
+                              message = str(e))
         if response.status_code != 200:
             self._handle_error(url, response.status_code, response.content)
         self._last_response = response
@@ -122,12 +122,10 @@ class HTTPClient(object):
         else:
             return self._req("POST", url, None, params)
 
-    def put(self, url, data, params = None, binary = False):
+    def put(self, url, data, params = None,
+            content_type = "application/octet-stream"):
         """Simple PUT (parameters in URL, data in body)"""
-        if binary:
-            h = { 'Content-type': 'application/octet-stream' }
-        else:
-            h = { 'Content-type': 'text/plain; charset=utf-8' }
+        h = { 'Content-type': content_type }
         return self._req("PUT", url, query = params, body = data, headers = h)
 
     # Generator versions that return data one line at a time.
@@ -156,7 +154,7 @@ class HTTPClient(object):
                     pending = tmp[-1]
                 for line in lines:
                     yield line
-            if pending is not None: # pragma: no cover (missing newline)
+            if pending is not None:
                 yield pending
 
         # Yield the chunks or lines as requested
@@ -165,11 +163,11 @@ class HTTPClient(object):
                 yield chunk
         elif isjson:
             for line in lines(response.iter_content(chunk_size = 1),
-                              ending = '\r\n'):
+                              ending = b'\r\n'):
                 yield json.loads(line)
         else:
             for line in lines(response.iter_content(chunk_size = 65536),
-                              ending = '\n'):
+                              ending = b'\n'):
                 yield line
 
     def get_gen(self, url, params = None, binary = False):

nilmdb/client/numpyclient.py

@@ -12,20 +12,24 @@ import contextlib
 from nilmdb.utils.time import timestamp_to_string, string_to_timestamp
 
 import numpy
-import cStringIO
+import io
 
 def layout_to_dtype(layout):
     ltype = layout.split('_')[0]
     lcount = int(layout.split('_')[1])
     if ltype.startswith('int'):
-        atype = '<i' + str(int(ltype[3:]) / 8)
+        atype = '<i' + str(int(ltype[3:]) // 8)
     elif ltype.startswith('uint'):
-        atype = '<u' + str(int(ltype[4:]) / 8)
+        atype = '<u' + str(int(ltype[4:]) // 8)
     elif ltype.startswith('float'):
-        atype = '<f' + str(int(ltype[5:]) / 8)
+        atype = '<f' + str(int(ltype[5:]) // 8)
     else:
         raise ValueError("bad layout")
-    return numpy.dtype([('timestamp', '<i8'), ('data', atype, lcount)])
+    if lcount == 1:
+        dtype = [('timestamp', '<i8'), ('data', atype)]
+    else:
+        dtype = [('timestamp', '<i8'), ('data', atype, lcount)]
+    return numpy.dtype(dtype)
 
 class NumpyClient(nilmdb.client.client.Client):
     """Subclass of nilmdb.client.Client that adds additional methods for
@@ -70,14 +74,14 @@ class NumpyClient(nilmdb.client.client.Client):
 
             # See if we have enough to make the requested Numpy array
             while total_len >= maxsize:
-                assembled = "".join(chunks)
+                assembled = b"".join(chunks)
                 total_len -= maxsize
                 chunks = [ assembled[maxsize:] ]
                 block = assembled[:maxsize]
                 yield to_numpy(block)
 
         if total_len:
-            yield to_numpy("".join(chunks))
+            yield to_numpy(b"".join(chunks))
 
     @contextlib.contextmanager
     def stream_insert_numpy_context(self, path, start = None, end = None,

nilmdb/cmdline/cmdline.py

@@ -3,7 +3,7 @@
 import nilmdb.client
 
 from nilmdb.utils.printf import *
-from nilmdb.utils import datetime_tz
+import datetime_tz
 import nilmdb.utils.time
 
 import sys
@@ -12,10 +12,7 @@ import argparse
 from argparse import ArgumentDefaultsHelpFormatter as def_form
 import signal
 
-try: # pragma: no cover
-    import argcomplete
-except ImportError: # pragma: no cover
-    argcomplete = None
+import argcomplete
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
@@ -40,7 +37,7 @@ class JimArgumentParser(argparse.ArgumentParser):
         self.print_usage(sys.stderr)
         self.exit(2, sprintf("error: %s\n", message))
 
-class Complete(object): # pragma: no cover
+class Complete(object):
     # Completion helpers, for using argcomplete (see
     # extras/nilmtool-bash-completion.sh)
     def escape(self, s):
@@ -80,11 +77,7 @@ class Complete(object): # pragma: no cover
         if not path:
             return []
         results = []
-        # prefix comes in as UTF-8, but results need to be Unicode,
-        # weird.  Still doesn't work in all cases, but that's bugs in
-        # argcomplete.
-        prefix = nilmdb.utils.unicode.decode(prefix)
-        for (k,v) in client.stream_get_metadata(path).iteritems():
+        for (k,v) in client.stream_get_metadata(path).items():
             kv = self.escape(k + '=' + v)
             if kv.startswith(prefix):
                 results.append(kv)
@@ -94,16 +87,11 @@ class Cmdline(object):
 
     def __init__(self, argv = None):
         self.argv = argv or sys.argv[1:]
-        try:
-            # Assume command line arguments are encoded with stdin's encoding,
-            # and reverse it.  Won't be needed in Python 3, but for now..
-            self.argv = [ x.decode(sys.stdin.encoding) for x in self.argv ]
-        except Exception: # pragma: no cover
-            pass
         self.client = None
         self.def_url = os.environ.get("NILMDB_URL", "http://localhost/nilmdb/")
         self.subcmd = {}
         self.complete = Complete()
+        self.complete_output_stream = None # overridden by test suite
 
     def arg_time(self, toparse):
         """Parse a time string argument"""
@@ -131,7 +119,7 @@ class Cmdline(object):
                            ).completer = self.complete.url
 
         sub = self.parser.add_subparsers(
-            title="Commands", dest="command",
+            title="Commands", dest="command", required=True,
             description="Use 'help command' or 'command --help' for more "
             "details on a particular command.")
 
@@ -148,10 +136,7 @@ class Cmdline(object):
     def run(self):
         # Set SIGPIPE to its default handler -- we don't need Python
         # to catch it for us.
-        try:
-            signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-        except ValueError: # pragma: no cover
-            pass
+        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
         # Clear cached timezone, so that we can pick up timezone changes
         # while running this from the test suite.
@@ -159,8 +144,8 @@ class Cmdline(object):
 
         # Run parser
         self.parser_setup()
-        if argcomplete: # pragma: no cover
-            argcomplete.autocomplete(self.parser)
+        argcomplete.autocomplete(self.parser, exit_method=sys.exit,
+                                 output_stream=self.complete_output_stream)
         self.args = self.parser.parse_args(self.argv)
 
         # Run arg verify handler if there is one

nilmdb/cmdline/extract.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+
 from nilmdb.utils.printf import *
 import nilmdb.client
 import sys
@@ -41,9 +41,8 @@ def setup(self, sub):
     return cmd
 
 def cmd_extract_verify(self):
-    if self.args.start is not None and self.args.end is not None:
-        if self.args.start > self.args.end:
-            self.parser.error("start is after end")
+    if self.args.start > self.args.end:
+        self.parser.error("start is after end")
 
     if self.args.binary:
         if (self.args.bare or self.args.annotate or self.args.markup or
@@ -69,9 +68,9 @@ def cmd_extract(self):
 
     printed = False
     if self.args.binary:
-        printer = sys.stdout.write
+        printer = sys.stdout.buffer.write
     else:
-        printer = print
+        printer = lambda x: print(x.decode('utf-8'))
     bare = self.args.bare
     count = self.args.count
     for dataline in self.client.stream_extract(self.args.path,
@@ -83,7 +82,7 @@ def cmd_extract(self):
         if bare and not count:
             # Strip timestamp (first element).  Doesn't make sense
             # if we are only returning a count.
-            dataline = ' '.join(dataline.split(' ')[1:])
+            dataline = b' '.join(dataline.split(b' ')[1:])
         printer(dataline)
         printed = True
     if not printed:

nilmdb/cmdline/insert.py

@@ -87,7 +87,7 @@ def cmd_insert(self):
     try:
         filename = arg.file
         if filename == '-':
-            infile = sys.stdin
+            infile = sys.stdin.buffer
         else:
             try:
                 infile = open(filename, "rb")
@@ -104,7 +104,7 @@ def cmd_insert(self):
         if arg.timestamp:
             data = timestamper.TimestamperRate(infile, arg.start, arg.rate)
         else:
-            data = iter(lambda: infile.read(1048576), '')
+            data = iter(lambda: infile.read(1048576), b'')
 
         # Print info
         if not arg.quiet:

nilmdb/cmdline/metadata.py

@@ -41,10 +41,10 @@ def cmd_metadata(self):
     if self.args.set is not None or self.args.update is not None:
         # Either set, or update
         if self.args.set is not None:
-            keyvals = map(nilmdb.utils.unicode.decode, self.args.set)
+            keyvals = self.args.set
             handler = self.client.stream_set_metadata
         else:
-            keyvals = map(nilmdb.utils.unicode.decode, self.args.update)
+            keyvals = self.args.update
             handler = self.client.stream_update_metadata
 
         # Extract key=value pairs
@@ -64,7 +64,7 @@ def cmd_metadata(self):
         # Delete (by setting values to empty strings)
         keys = None
         if self.args.delete:
-            keys = map(nilmdb.utils.unicode.decode, self.args.delete)
+            keys = list(self.args.delete)
         try:
             data = self.client.stream_get_metadata(self.args.path, keys)
             for key in data:
@@ -76,7 +76,7 @@ def cmd_metadata(self):
         # Get (or unspecified)
         keys = None
         if self.args.get:
-            keys = map(nilmdb.utils.unicode.decode, self.args.get)
+            keys = list(self.args.get)
         try:
             data = self.client.stream_get_metadata(self.args.path, keys)
         except nilmdb.client.ClientError as e:
@@ -85,6 +85,4 @@ def cmd_metadata(self):
             # Print nonexistant keys as having empty value
             if value is None:
                 value = ""
-            printf("%s=%s\n",
-                   nilmdb.utils.unicode.encode(key),
-                   nilmdb.utils.unicode.encode(value))
+            printf("%s=%s\n", key, value)

nilmdb/fsck/__init__.py

@@ -1,5 +1,5 @@
 """nilmdb.fsck"""
 
-from __future__ import absolute_import
+
 
 from nilmdb.fsck.fsck import Fsck

nilmdb/fsck/fsck.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+raise Exception("todo: fix path bytes issues")
+
 """Check database consistency, with some ability to fix problems.
 This should be able to fix cases where a database gets corrupted due
 to unexpected system shutdown, and detect other cases that may cause
@@ -21,7 +23,7 @@ import progressbar
 import re
 import time
 import shutil
-import cPickle as pickle
+import pickle
 import numpy
 
 class FsckError(Exception):
@@ -179,7 +181,7 @@ class Fsck(object):
     ### Check streams and basic interval overlap
 
     def check_streams(self):
-        ids = self.stream_path.keys()
+        ids = list(self.stream_path.keys())
         log("checking %s streams\n", "{:,d}".format(len(ids)))
         with Progress(len(ids)) as pbar:
             for i, sid in enumerate(ids):
@@ -187,7 +189,7 @@ class Fsck(object):
                 path = self.stream_path[sid]
 
                 # unique path, valid layout
-                if self.stream_path.values().count(path) != 1:
+                if list(self.stream_path.values()).count(path) != 1:
                     raise FsckError("duplicated path %s", path)
                 layout = self.stream_layout[sid].split('_')[0]
                 if layout not in ('int8', 'int16', 'int32', 'int64',
@@ -269,7 +271,7 @@ class Fsck(object):
         for subdir in subdirs:
             # Find all files in that dir
             subpath = os.path.join(bulk, subdir)
-            files = filter(regex.search, os.listdir(subpath))
+            files = list(filter(regex.search, os.listdir(subpath)))
             if not files:
                 self.fix_empty_subdir(subpath)
                 raise RetryFsck
@@ -315,7 +317,7 @@ class Fsck(object):
     ### Check interval endpoints
 
     def check_intervals(self):
-        total_ints = sum(len(x) for x in self.stream_interval.values())
+        total_ints = sum(len(x) for x in list(self.stream_interval.values()))
         log("checking %s intervals\n", "{:,d}".format(total_ints))
         done = 0
         with Progress(total_ints) as pbar:
@@ -398,7 +400,7 @@ class Fsck(object):
 
     def check_data(self):
         total_rows = sum(sum((y[3] - y[2]) for y in x)
-                         for x in self.stream_interval.values())
+                         for x in list(self.stream_interval.values()))
         log("checking %s rows of data\n", "{:,d}".format(total_rows))
         done = 0
         with Progress(total_rows) as pbar:

nilmdb/scripts/nilmdb_fsck.py

@@ -10,8 +10,9 @@ def main():
 
     parser = argparse.ArgumentParser(
         description = 'Check database consistency',
-        formatter_class = argparse.ArgumentDefaultsHelpFormatter,
-        version = nilmdb.__version__)
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-v", "--version", action="version",
+                        version = nilmdb.__version__)
     parser.add_argument("-f", "--fix", action="store_true",
                         default=False, help = 'Fix errors when possible '
                         '(which may involve removing data)')

nilmdb/scripts/nilmdb_server.py

@@ -4,14 +4,18 @@ import nilmdb.server
 import argparse
 import os
 import socket
+import cherrypy
+import sys
 
 def main():
     """Main entry point for the 'nilmdb-server' command line script"""
 
     parser = argparse.ArgumentParser(
         description = 'Run the NilmDB server',
-        formatter_class = argparse.ArgumentDefaultsHelpFormatter,
-        version = nilmdb.__version__)
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument("-v", "--version", action="version",
+                        version = nilmdb.__version__)
 
     group = parser.add_argument_group("Standard options")
     group.add_argument('-a', '--address',
@@ -39,47 +43,49 @@ def main():
     db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database)
 
     # Configure the server
-    if args.quiet:
-        embedded = True
-    else:
-        embedded = False
+    if not args.quiet:
+        cherrypy._cpconfig.environments['embedded']['log.screen'] = True
     server = nilmdb.server.Server(db,
                                   host = args.address,
                                   port = args.port,
-                                  embedded = embedded,
                                   force_traceback = args.traceback)
 
     # Print info
     if not args.quiet:
-        print "Version: %s" % nilmdb.__version__
-        print "Database: %s" % (os.path.realpath(args.database))
+        print("Version: %s" % nilmdb.__version__)
+        print("Database: %s" % (os.path.realpath(args.database)))
         if args.address == '0.0.0.0' or args.address == '::':
             host = socket.getfqdn()
         else:
             host = args.address
-        print "Server URL: http://%s:%d/" % ( host, args.port)
-        print "----"
+        print("Server URL: http://%s:%d/" % ( host, args.port))
+        print("----")
 
     # Run it
-    if args.yappi:
-        print "Running in yappi"
-        try:
-            import yappi
-            yappi.start()
+    try:
+        if args.yappi:
+            print("Running in yappi")
+            try:
+                import yappi
+                yappi.start()
+                server.start(blocking = True)
+            finally:
+                yappi.stop()
+                stats = yappi.get_func_stats()
+                stats.sort("ttot")
+                stats.print_all()
+                from IPython import embed
+                embed(header = "Use the `yappi` or `stats` object to explore "
+                      "further, quit to exit")
+        else:
             server.start(blocking = True)
-        finally:
-            yappi.stop()
-            yappi.print_stats(sort_type = yappi.SORTTYPE_TTOT, limit = 50)
-            from IPython import embed
-            embed(header = "Use the yappi object to explore further, "
-                  "quit to exit")
-    else:
-        server.start(blocking = True)
-
-    # Clean up
-    if not args.quiet:
-        print "Closing database"
-        db.close()
+    except nilmdb.server.serverutil.CherryPyExit as e:
+        print("Exiting due to CherryPy error", file=sys.stderr)
+        raise
+    finally:
+        if not args.quiet:
+            print("Closing database")
+            db.close()
 
 if __name__ == "__main__":
     main()

nilmdb/server/__init__.py

@@ -1,20 +1,8 @@
 """nilmdb.server"""
 
-from __future__ import absolute_import
-
-# Try to set up pyximport to automatically rebuild Cython modules.  If
-# this doesn't work, it's OK, as long as the modules were built externally.
-# (e.g. python setup.py build_ext --inplace)
-try: # pragma: no cover
-    import Cython
-    import distutils.version
-    if (distutils.version.LooseVersion(Cython.__version__) <
-        distutils.version.LooseVersion("0.17")): # pragma: no cover
-        raise ImportError("Cython version too old")
-    import pyximport
-    pyximport.install(inplace = True, build_in_temp = False)
-except (ImportError, TypeError): # pragma: no cover
-    pass
+# Set up pyximport to automatically rebuild Cython modules if needed.
+import pyximport
+pyximport.install(inplace = True, build_in_temp = False)
 
 from nilmdb.server.nilmdb import NilmDB
 from nilmdb.server.server import Server, wsgi_application

nilmdb/server/bulkdata.py

@@ -2,14 +2,14 @@
 
 # Need absolute_import so that "import nilmdb" won't pull in
 # nilmdb.py, but will pull the parent nilmdb module instead.
-from __future__ import absolute_import
-from __future__ import division
+
+
 from nilmdb.utils.printf import *
 from nilmdb.utils.time import timestamp_to_string as timestamp_to_string
 import nilmdb.utils
 
 import os
-import cPickle as pickle
+import pickle
 import re
 import sys
 import tempfile
@@ -25,25 +25,28 @@ fd_cache_size = 8
 @nilmdb.utils.must_close(wrap_verify = False)
 class BulkData(object):
     def __init__(self, basepath, **kwargs):
-        self.basepath = basepath
-        self.root = os.path.join(self.basepath, "data")
-        self.lock = self.root + ".lock"
+        if isinstance(basepath, str):
+            self.basepath = self._encode_filename(basepath)
+        else:
+            self.basepath = basepath
+        self.root = os.path.join(self.basepath, b"data")
+        self.lock = self.root + b".lock"
         self.lockfile = None
 
         # Tuneables
-        if "file_size" in kwargs:
+        if "file_size" in kwargs and kwargs["file_size"] is not None:
             self.file_size = kwargs["file_size"]
         else:
             # Default to approximately 128 MiB per file
             self.file_size = 128 * 1024 * 1024
 
-        if "files_per_dir" in kwargs:
+        if "files_per_dir" in kwargs and kwargs["files_per_dir"] is not None:
             self.files_per_dir = kwargs["files_per_dir"]
         else:
             # 32768 files per dir should work even on FAT32
             self.files_per_dir = 32768
 
-        if "initial_nrows" in kwargs:
+        if "initial_nrows" in kwargs and kwargs["initial_nrows"] is not None:
             self.initial_nrows = kwargs["initial_nrows"]
         else:
             # First row is 0
@@ -56,7 +59,8 @@ class BulkData(object):
         # Create the lock
         self.lockfile = open(self.lock, "w")
         if not nilmdb.utils.lock.exclusive_lock(self.lockfile):
-            raise IOError('database at "' + self.basepath +
+            raise IOError('database at "' +
+                          self._decode_filename(self.basepath) +
                           '" is already locked by another process')
 
     def close(self):
@@ -66,21 +70,21 @@ class BulkData(object):
             self.lockfile.close()
             try:
                 os.unlink(self.lock)
-            except OSError: # pragma: no cover
+            except OSError:
                 pass
             self.lockfile = None
 
     def _encode_filename(self, path):
-        # Encode all paths to UTF-8, regardless of sys.getfilesystemencoding(),
-        # because we want to be able to represent all code points and the user
-        # will never be directly exposed to filenames.  We can then do path
-        # manipulations on the UTF-8 directly.
-        if isinstance(path, unicode):
-            return path.encode('utf-8')
-        return path
+        # Translate unicode strings to raw bytes, if needed.  We
+        # always manipulate paths internally as bytes.
+        return path.encode('utf-8')
+
+    def _decode_filename(self, path):
+        # Translate raw bytes to unicode strings, escaping if needed
+        return path.decode('utf-8', errors='backslashreplace')
 
     def _create_check_ospath(self, ospath):
-        if ospath[-1] == '/':
+        if ospath[-1:] == b'/':
             raise ValueError("invalid path; should not end with a /")
         if Table.exists(ospath):
             raise ValueError("stream already exists at this path")
@@ -97,13 +101,13 @@ class BulkData(object):
         don't exist.  Returns a list of elements that got created."""
         path = self._encode_filename(unicodepath)
 
-        if path[0] != '/':
-            raise ValueError("paths must start with /")
-        [ group, node ] = path.rsplit("/", 1)
-        if group == '':
+        if path[0:1] != b'/':
+            raise ValueError("paths must start with / ")
+        [ group, node ] = path.rsplit(b"/", 1)
+        if group == b'':
             raise ValueError("invalid path; path must contain at least one "
                              "folder")
-        if node == '':
+        if node == b'':
             raise ValueError("invalid path; should not end with a /")
         if not Table.valid_path(path):
             raise ValueError("path name is invalid or contains reserved words")
@@ -114,7 +118,7 @@ class BulkData(object):
         # os.path.join)
 
         # Make directories leading up to this one
-        elements = path.lstrip('/').split('/')
+        elements = path.lstrip(b'/').split(b'/')
         made_dirs = []
         try:
             # Make parent elements
@@ -126,14 +130,10 @@ class BulkData(object):
                     os.mkdir(ospath)
                     made_dirs.append(ospath)
         except Exception as e:
-            # Try to remove paths that we created; ignore errors
-            exc_info = sys.exc_info()
-            for ospath in reversed(made_dirs): # pragma: no cover (hard to hit)
-                try:
-                    os.rmdir(ospath)
-                except OSError:
-                    pass
-            raise exc_info[1], None, exc_info[2]
+            # Remove paths that we created
+            for ospath in reversed(made_dirs):
+                os.rmdir(ospath)
+            raise
 
         return elements
 
@@ -168,7 +168,7 @@ class BulkData(object):
                 os.rmdir(ospath)
             except OSError:
                 pass
-            raise exc_info[1], None, exc_info[2]
+            raise exc_info[1].with_traceback(exc_info[2])
 
         # Success
         return
@@ -176,8 +176,8 @@ class BulkData(object):
     def _remove_leaves(self, unicodepath):
         """Remove empty directories starting at the leaves of unicodepath"""
         path = self._encode_filename(unicodepath)
-        elements = path.lstrip('/').split('/')
-        for i in reversed(range(len(elements))):
+        elements = path.lstrip(b'/').split(b'/')
+        for i in reversed(list(range(len(elements)))):
             ospath = os.path.join(self.root, *elements[0:i+1])
             try:
                 os.rmdir(ospath)
@@ -191,9 +191,9 @@ class BulkData(object):
         newpath = self._encode_filename(newunicodepath)
 
         # Get OS paths
-        oldelements = oldpath.lstrip('/').split('/')
+        oldelements = oldpath.lstrip(b'/').split(b'/')
         oldospath = os.path.join(self.root, *oldelements)
-        newelements = newpath.lstrip('/').split('/')
+        newelements = newpath.lstrip(b'/').split(b'/')
         newospath = os.path.join(self.root, *newelements)
 
         # Basic checks
@@ -204,8 +204,8 @@ class BulkData(object):
         self.getnode.cache_remove(self, oldunicodepath)
 
         # Move the table to a temporary location
-        tmpdir = tempfile.mkdtemp(prefix = "rename-", dir = self.root)
-        tmppath = os.path.join(tmpdir, "table")
+        tmpdir = tempfile.mkdtemp(prefix = b"rename-", dir = self.root)
+        tmppath = os.path.join(tmpdir, b"table")
         os.rename(oldospath, tmppath)
 
         try:
@@ -233,7 +233,7 @@ class BulkData(object):
         path = self._encode_filename(unicodepath)
 
         # Get OS path
-        elements = path.lstrip('/').split('/')
+        elements = path.lstrip(b'/').split(b'/')
         ospath = os.path.join(self.root, *elements)
 
         # Remove Table object from cache
@@ -258,7 +258,7 @@ class BulkData(object):
         """Return a Table object corresponding to the given database
         path, which must exist."""
         path = self._encode_filename(unicodepath)
-        elements = path.lstrip('/').split('/')
+        elements = path.lstrip(b'/').split(b'/')
         ospath = os.path.join(self.root, *elements)
         return Table(ospath, self.initial_nrows)
 
@@ -271,12 +271,12 @@ class Table(object):
     @classmethod
     def valid_path(cls, root):
         """Return True if a root path is a valid name"""
-        return "_format" not in root.split("/")
+        return b"_format" not in root.split(b"/")
 
     @classmethod
     def exists(cls, root):
         """Return True if a table appears to exist at this OS path"""
-        return os.path.isfile(os.path.join(root, "_format"))
+        return os.path.isfile(os.path.join(root, b"_format"))
 
     @classmethod
     def create(cls, root, layout, file_size, files_per_dir):
@@ -293,7 +293,7 @@ class Table(object):
                 "files_per_dir": files_per_dir,
                 "layout": layout,
                 "version": 3 }
-        with open(os.path.join(root, "_format"), "wb") as f:
+        with open(os.path.join(root, b"_format"), "wb") as f:
             pickle.dump(fmt, f, 2)
 
     # Normal methods
@@ -303,10 +303,10 @@ class Table(object):
         self.initial_nrows = initial_nrows
 
         # Load the format
-        with open(os.path.join(self.root, "_format"), "rb") as f:
+        with open(os.path.join(self.root, b"_format"), "rb") as f:
             fmt = pickle.load(f)
 
-        if fmt["version"] != 3: # pragma: no cover
+        if fmt["version"] != 3:
             # Old versions used floating point timestamps, which aren't
             # valid anymore.
             raise NotImplementedError("old version " + str(fmt["version"]) +
@@ -336,7 +336,7 @@ class Table(object):
         # greater than the row number of any piece of data that
         # currently exists, not necessarily all data that _ever_
         # existed.
-        regex = re.compile("^[0-9a-f]{4,}$")
+        regex = re.compile(b"^[0-9a-f]{4,}$")
 
         # Find the last directory.  We sort and loop through all of them,
         # starting with the numerically greatest, because the dirs could be
@@ -348,8 +348,8 @@ class Table(object):
         for subdir in subdirs:
             # Now find the last file in that dir
             path = os.path.join(self.root, subdir)
-            files = filter(regex.search, os.listdir(path))
-            if not files: # pragma: no cover (shouldn't occur)
+            files = list(filter(regex.search, os.listdir(path)))
+            if not files:
                 # Empty dir: try the next one
                 continue
 
@@ -380,8 +380,8 @@ class Table(object):
         filenum = row // self.rows_per_file
         # It's OK if these format specifiers are too short; the filenames
         # will just get longer but will still sort correctly.
-        dirname = sprintf("%04x", filenum // self.files_per_dir)
-        filename = sprintf("%04x", filenum % self.files_per_dir)
+        dirname = sprintf(b"%04x", filenum // self.files_per_dir)
+        filename = sprintf(b"%04x", filenum % self.files_per_dir)
         offset = (row % self.rows_per_file) * self.row_size
         count = self.rows_per_file - (row % self.rows_per_file)
         return (dirname, filename, offset, count)
@@ -389,7 +389,7 @@ class Table(object):
     def _row_from_offset(self, subdir, filename, offset):
         """Return the row number that corresponds to the given
         'subdir/filename' and byte-offset within that file."""
-        if (offset % self.row_size) != 0: # pragma: no cover
+        if (offset % self.row_size) != 0:
             # this shouldn't occur, unless there is some corruption somewhere
             raise ValueError("file offset is not a multiple of data size")
         filenum = int(subdir, 16) * self.files_per_dir + int(filename, 16)
@@ -436,6 +436,8 @@ class Table(object):
         are non-monotonic, or don't fall between 'start' and 'end',
         a ValueError is raised.
 
+        Note that data is always of 'bytes' type.
+
         If 'binary' is True, the data should be in raw binary format
         instead: little-endian, matching the current table's layout,
         including the int64 timestamp.
@@ -476,9 +478,9 @@ class Table(object):
                         if binary:
                             raise IndexError
                         bad = data.splitlines()[linenum-1]
-                        bad += '\n' + ' ' * (colnum - 1) + '^'
+                        bad += b'\n' + b' ' * (colnum - 1) + b'^'
                     except IndexError:
-                        bad = ""
+                        bad = b""
                     if errtype == rocket.ERR_NON_MONOTONIC:
                         err = "timestamp is not monotonically increasing"
                     elif errtype == rocket.ERR_OUT_OF_INTERVAL:
@@ -492,8 +494,9 @@ class Table(object):
                                           timestamp_to_string(end))
                     else:
                         err = str(obj)
+                    bad_str = bad.decode('utf-8', errors='backslashreplace')
                     raise ValueError("error parsing input data: " +
-                                     where + err + "\n" + bad)
+                                     where + err + "\n" + bad_str)
                 tot_rows += added_rows
         except Exception:
             # Some failure, so try to roll things back by truncating or
@@ -556,7 +559,7 @@ class Table(object):
         # file.  Only when the list covers the entire extent of the
         # file will that file be removed.
         datafile = os.path.join(self.root, subdir, filename)
-        cachefile = datafile + ".removed"
+        cachefile = datafile + b".removed"
         try:
             with open(cachefile, "rb") as f:
                 ranges = pickle.load(f)
@@ -583,8 +586,9 @@ class Table(object):
                 # Not connected; append previous and start again
                 merged.append(prev)
                 prev = new
-        if prev is not None:
-            merged.append(prev)
+        # Last range we were looking at goes into the file.  We know
+        # there was at least one (the one we just removed).