Huge cleanup of namespaces, modules, packages, imports.

Now nilmdb.client, nilmdb.server, nilmdb.cmdline, and nilmdb.utils
are each their own modules, and there is a little bit more of a
logical separation between them.  Various changes scattered throughout
to fix naming (for example, nilmdb.nilmdb.NilmDBError is now
nilmdb.server.errors.NilmDBError).

Reduced usage of "from __future__ import absolute_import" as much
as possible.  It's still needed for the functions in the nilmdb/server
directory to be able to import the nilmdb module rather than the
nilmdb.py script.

This should hopefully ease future packaging a bit.

nilmdb/__init__.py

@@ -1,12 +1,4 @@
 """Main NilmDB import"""
 
-from .nilmdb import NilmDB
-from .server import Server
-from .client import Client
-
-import pyximport; pyximport.install()
-import layout
-import interval
-
-import cmdline
-
+from server import NilmDB, Server
+from client import Client

nilmdb/client/__init__.py

@@ -0,0 +1,4 @@
+"""nilmdb.client"""
+
+from .client import Client
+from .errors import *

nilmdb/client.py → nilmdb/client/client.py

@@ -2,7 +2,9 @@
 
 """Class for performing HTTP client requests via libcurl"""
 
-from __future__ import absolute_import
+import nilmdb
+import nilmdb.utils
+import nilmdb.client.httpclient
 from nilmdb.utils.printf import *
 
 import time
@@ -12,12 +14,6 @@ import os
 import simplejson as json
 import itertools
 
-import nilmdb.utils
-import nilmdb.httpclient
-
-# Other functions expect to see these in the nilmdb.client namespace
-from nilmdb.httpclient import ClientError, ServerError, Error
-
 version = "1.0"
 
 def float_to_string(f):
@@ -30,7 +26,7 @@ class Client(object):
     client_version = version
 
     def __init__(self, url):
-        self.http = nilmdb.httpclient.HTTPClient(url)
+        self.http = nilmdb.client.httpclient.HTTPClient(url)
 
     def _json_param(self, data):
         """Return compact json-encoded version of parameter"""

nilmdb/client/errors.py

@@ -0,0 +1,33 @@
+"""HTTP client errors"""
+
+from nilmdb.utils.printf import *
+
+class Error(Exception):
+    """Base exception for both ClientError and ServerError responses"""
+    def __init__(self,
+                 status = "Unspecified error",
+                 message = None,
+                 url = None,
+                 traceback = None):
+        Exception.__init__(self, status)
+        self.status = status     # e.g. "400 Bad Request"
+        self.message = message   # textual message from the server
+        self.url = url           # URL we were requesting
+        self.traceback = traceback # server traceback, if available
+    def _format_error(self, show_url):
+        s = sprintf("[%s]", self.status)
+        if self.message:
+            s += sprintf(" %s", self.message)
+        if show_url and self.url: # pragma: no cover
+            s += sprintf(" (%s)", self.url)
+        if self.traceback: # pragma: no cover
+            s += sprintf("\nServer traceback:\n%s", self.traceback)
+        return s
+    def __str__(self):
+        return self._format_error(show_url = False)
+    def __repr__(self): # pragma: no cover
+        return self._format_error(show_url = True)
+class ClientError(Error):
+    pass
+class ServerError(Error):
+    pass

nilmdb/httpclient.py → nilmdb/client/httpclient.py

@@ -1,8 +1,9 @@
 """HTTP client library"""
 
-from __future__ import absolute_import
-from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.utils
+from nilmdb.utils.printf import *
+from nilmdb.client.errors import *
 
 import time
 import sys
@@ -13,39 +14,6 @@ import urlparse
 import pycurl
 import cStringIO
 
-class Error(Exception):
-    """Base exception for both ClientError and ServerError responses"""
-    def __init__(self,
-                 status = "Unspecified error",
-                 message = None,
-                 url = None,
-                 traceback = None):
-        Exception.__init__(self, status)
-        self.status = status     # e.g. "400 Bad Request"
-        self.message = message   # textual message from the server
-        self.url = url           # URL we were requesting
-        self.traceback = traceback # server traceback, if available
-    def __str__(self):
-        s = sprintf("[%s]", self.status)
-        if self.message:
-            s += sprintf(" %s", self.message)
-        if self.traceback: # pragma: no cover
-            s += sprintf("\nServer traceback:\n%s", self.traceback)
-        return s
-    def __repr__(self): # pragma: no cover
-        s = sprintf("[%s]", self.status)
-        if self.message:
-            s += sprintf(" %s", self.message)
-        if self.url:
-            s += sprintf(" (%s)", self.url)
-        if self.traceback:
-            s += sprintf("\nServer traceback:\n%s", self.traceback)
-        return s
-class ClientError(Error):
-    pass
-class ServerError(Error):
-    pass
-
 class HTTPClient(object):
     """Class to manage and perform HTTP requests from the client"""
     def __init__(self, baseurl = ""):

nilmdb/cmdline/__init__.py

@@ -1 +1,3 @@
+"""nilmdb.cmdline"""
+
 from .cmdline import Cmdline

nilmdb/cmdline/cmdline.py

@@ -1,8 +1,7 @@
 """Command line client functionality"""
 
-from __future__ import absolute_import
+import nilmdb
 from nilmdb.utils.printf import *
-import nilmdb.client
 
 import datetime_tz
 import dateutil.parser

nilmdb/cmdline/create.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 import textwrap
 

nilmdb/cmdline/destroy.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form

nilmdb/cmdline/extract.py

@@ -1,6 +1,6 @@
-from __future__ import absolute_import
 from __future__ import print_function
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 import sys
 

nilmdb/cmdline/info.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
 
 from argparse import ArgumentDefaultsHelpFormatter as def_form

nilmdb/cmdline/insert.py

@@ -1,7 +1,7 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
-import nilmdb.timestamper
+import nilmdb.utils.timestamper as timestamper
 
 import sys
 
@@ -69,7 +69,7 @@ def cmd_insert(self):
 
         # Build a timestamper for this file
         if self.args.none:
-            ts = nilmdb.timestamper.TimestamperNull(infile)
+            ts = timestamper.TimestamperNull(infile)
         else:
             if self.args.start:
                 start = self.args.start
@@ -84,7 +84,7 @@ def cmd_insert(self):
                 self.die("error: --rate is needed, but was not specified")
             rate = self.args.rate
 
-            ts = nilmdb.timestamper.TimestamperRate(infile, start, rate)
+            ts = timestamper.TimestamperRate(infile, start, rate)
 
         # Print info
         if not self.args.quiet:

nilmdb/cmdline/list.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 
 import fnmatch

nilmdb/cmdline/metadata.py

@@ -1,5 +1,5 @@
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 
 def setup(self, sub):

nilmdb/cmdline/remove.py

@@ -1,6 +1,5 @@
-from __future__ import absolute_import
-from __future__ import print_function
 from nilmdb.utils.printf import *
+import nilmdb
 import nilmdb.client
 import sys
 

nilmdb/server/__init__.py

@@ -0,0 +1,14 @@
+"""nilmdb.server"""
+
+# Try to set up pyximport to automatically rebuild Cython modules.  If
+# this doesn't work, it's OK, as long as the modules were built externally.
+try:
+    import pyximport
+    pyximport.install()
+    import layout
+except: # pragma: no cover
+    pass
+
+from .nilmdb import NilmDB
+from .server import Server
+from .errors import *

nilmdb/bulkdata.py → nilmdb/server/bulkdata.py

@@ -1,5 +1,7 @@
 # Fixed record size bulk data storage
 
+# Need absolute_import so that "import nilmdb" won't pull in
+# nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
 from __future__ import division
 import nilmdb
@@ -75,7 +77,7 @@ class BulkData(object):
 
         # Get layout, and build format string for struct module
         try:
-            layout = nilmdb.layout.get_named(layout_name)
+            layout = nilmdb.server.layout.get_named(layout_name)
             struct_fmt = '<d'  # Little endian, double timestamp
             struct_mapping = {
                 "int8": 'b',

nilmdb/server/errors.py

@@ -0,0 +1,12 @@
+"""Exceptions"""
+
+class NilmDBError(Exception):
+    """Base exception for NilmDB errors"""
+    def __init__(self, message = "Unspecified error"):
+        Exception.__init__(self, message)
+
+class StreamError(NilmDBError):
+    pass
+
+class OverlapError(NilmDBError):
+    pass

nilmdb/nilmdb.py → nilmdb/server/nilmdb.py

@@ -7,11 +7,15 @@ Object that represents a NILM database file.
 Manages both the SQL database and the table storage backend.
 """
 
-# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
-# but will pull the nilmdb module instead.
+# Need absolute_import so that "import nilmdb" won't pull in
+# nilmdb.py, but will pull the parent nilmdb module instead.
 from __future__ import absolute_import
 import nilmdb
 from nilmdb.utils.printf import *
+from nilmdb.server.interval import (Interval, DBInterval,
+                                    IntervalSet, IntervalError)
+from nilmdb.server import bulkdata
+from nilmdb.server.errors import *
 
 import sqlite3
 import time
@@ -20,12 +24,6 @@ import os
 import errno
 import bisect
 
-import pyximport
-pyximport.install()
-from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
-
-from . import bulkdata
-
 # Note about performance and transactions:
 #
 # Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
@@ -77,17 +75,6 @@ _sql_schema_updates = {
     """,
 }
 
-class NilmDBError(Exception):
-    """Base exception for NilmDB errors"""
-    def __init__(self, message = "Unspecified error"):
-        Exception.__init__(self, message)
-
-class StreamError(NilmDBError):
-    pass
-
-class OverlapError(NilmDBError):
-    pass
-
 @nilmdb.utils.must_close()
 class NilmDB(object):
     verbose = 0

nilmdb/server.py → nilmdb/server/server.py

@@ -1,10 +1,11 @@
 """CherryPy-based server for accessing NILM database via HTTP"""
 
-# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
-# but will pull the nilmdb module instead.
+# Need absolute_import so that "import nilmdb" won't pull in
+# nilmdb.py, but will pull the nilmdb module instead.
 from __future__ import absolute_import
-from nilmdb.utils.printf import *
 import nilmdb
+from nilmdb.utils.printf import *
+from nilmdb.server.errors import *
 
 import cherrypy
 import sys
@@ -14,8 +15,6 @@ import simplejson as json
 import decorator
 import traceback
 
-from nilmdb.nilmdb import NilmDBError
-
 try:
     import cherrypy
     cherrypy.tools.json_out
@@ -159,7 +158,7 @@ class Stream(NilmApp):
         matching the given keys."""
         try:
             data = self.db.stream_get_metadata(path)
-        except nilmdb.nilmdb.StreamError as e:
+        except nilmdb.server.nilmdb.StreamError as e:
             raise cherrypy.HTTPError("404 Not Found", e.message)
         if key is None:  # If no keys specified, return them all
             key = data.keys()
@@ -222,9 +221,9 @@ class Stream(NilmApp):
 
         # Parse the input data
         try:
-            parser = nilmdb.layout.Parser(layout)
+            parser = nilmdb.server.layout.Parser(layout)
             parser.parse(body)
-        except nilmdb.layout.ParserError as e:
+        except nilmdb.server.layout.ParserError as e:
             raise cherrypy.HTTPError("400 Bad Request",
                                      "error parsing input data: " +
                                      e.message)
@@ -249,7 +248,7 @@ class Stream(NilmApp):
         # Now do the nilmdb insert, passing it the parser full of data.
         try:
             result = self.db.stream_insert(path, start, end, parser.data)
-        except nilmdb.nilmdb.NilmDBError as e:
+        except NilmDBError as e:
             raise cherrypy.HTTPError("400 Bad Request", e.message)
 
         # Done
@@ -349,7 +348,7 @@ class Stream(NilmApp):
         layout = streams[0][1]
 
         # Get formatter
-        formatter = nilmdb.layout.Formatter(layout)
+        formatter = nilmdb.server.layout.Formatter(layout)
 
         @workaround_cp_bug_1200
         def content(start, end, count):

nilmdb/utils/diskusage.py

@@ -1,4 +1,3 @@
-import nilmdb
 import os
 from math import log
 

nilmdb/timestamper.py → nilmdb/utils/timestamper.py

@@ -1,6 +1,5 @@
 """File-like objects that add timestamps to the input lines"""
 
-from __future__ import absolute_import
 from nilmdb.utils.printf import *
 
 import time

nilmdb/utils/urllib.py

@@ -20,9 +20,6 @@ def urlencode(query):
             v = quote_plus(v)
             l.append(k + '=' + v)
         elif _is_unicode(v):
-            # is there a reasonable way to convert to ASCII?
-            # encode generates a string, but "replace" or "ignore"
-            # lose information and "strict" can raise UnicodeError
             v = quote_plus(v.encode("utf-8","strict"))
             l.append(k + '=' + v)
         else:

tests/test_bulkdata.py

@@ -2,8 +2,6 @@
 
 import nilmdb
 from nilmdb.utils.printf import *
-import nilmdb.bulkdata
-
 from nose.tools import *
 from nose.tools import assert_raises
 import itertools
@@ -12,7 +10,8 @@ from testutil.helpers import *
 
 testdb = "tests/bulkdata-testdb"
 
-from nilmdb.bulkdata import BulkData
+import nilmdb.server.bulkdata
+from nilmdb.server.bulkdata import BulkData
 
 class TestBulkData(object):
 

tests/test_client.py

@@ -2,6 +2,7 @@
 
 import nilmdb
 from nilmdb.utils.printf import *
+from nilmdb.utils import timestamper
 from nilmdb.client import ClientError, ServerError
 
 import datetime_tz
@@ -158,13 +159,13 @@ class TestClient(object):
         rate = 120
 
         # First try a nonexistent path
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/no-such-path", data)
         in_("404 Not Found", str(e.exception))
 
         # Now try reversed timestamps
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         data = reversed(list(data))
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
@@ -173,7 +174,7 @@ class TestClient(object):
 
         # Now try empty data (no server request made)
         empty = cStringIO.StringIO("")
-        data = nilmdb.timestamper.TimestamperRate(empty, start, 120)
+        data = timestamper.TimestamperRate(empty, start, 120)
         result = client.stream_insert("/newton/prep", data)
         eq_(result, None)
 
@@ -185,7 +186,7 @@ class TestClient(object):
         in_("no data provided", str(e.exception))
 
         # Specify start/end (starts too late)
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
                                           start + 5, start + 120)
@@ -194,7 +195,7 @@ class TestClient(object):
             str(e.exception))
 
         # Specify start/end (ends too early)
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data,
                                           start, start + 1)
@@ -205,7 +206,7 @@ class TestClient(object):
             str(e.exception))
 
         # Now do the real load
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         result = client.stream_insert("/newton/prep", data,
                                       start, start + 119.999777)
         eq_(result, "ok")
@@ -216,7 +217,7 @@ class TestClient(object):
         eq_(intervals, [[start, start + 119.999777]])
 
         # Try some overlapping data -- just insert it again
-        data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
+        data = timestamper.TimestamperRate(testfile, start, 120)
         with assert_raises(ClientError) as e:
             result = client.stream_insert("/newton/prep", data)
         in_("400 Bad Request", str(e.exception))

tests/test_interval.py

@@ -8,7 +8,8 @@ from nose.tools import *
 from nose.tools import assert_raises
 import itertools
 
-from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
+from nilmdb.server.interval import (Interval, DBInterval,
+                                    IntervalSet, IntervalError)
 
 from testutil.helpers import *
 import unittest

tests/test_layout.py

@@ -22,17 +22,17 @@ import unittest
 
 from testutil.helpers import *
 
-from nilmdb.layout import *
+from nilmdb.server.layout import *
 
 class TestLayouts(object):
     # Some nilmdb.layout tests.  Not complete, just fills in missing
     # coverage.
     def test_layouts(self):
-        x = nilmdb.layout.get_named("PrepData")
-        y = nilmdb.layout.get_named("float32_8")
+        x = nilmdb.server.layout.get_named("PrepData")
+        y = nilmdb.server.layout.get_named("float32_8")
         eq_(x.count, y.count)
         eq_(x.datatype, y.datatype)
-        y = nilmdb.layout.get_named("float32_7")
+        y = nilmdb.server.layout.get_named("float32_7")
         ne_(x.count, y.count)
         eq_(x.datatype, y.datatype)
 

tests/test_rbtree.py

@@ -6,7 +6,7 @@ from nilmdb.utils.printf import *
 from nose.tools import *
 from nose.tools import assert_raises
 
-from nilmdb.rbtree import RBTree, RBNode
+from nilmdb.server.rbtree import RBTree, RBNode
 
 from testutil.helpers import *
 import unittest

tests/test_timestamper.py

@@ -11,6 +11,8 @@ import cStringIO
 
 from testutil.helpers import *
 
+from nilmdb.utils import timestamper
+
 class TestTimestamper(object):
 
     # Not a very comprehensive test, but it's good enough.
@@ -27,20 +29,20 @@ class TestTimestamper(object):
 
         # full
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
+        ts = timestamper.TimestamperRate(input, start, 8000)
         foo = ts.readlines()
         eq_(foo, join(lines_out))
         in_("TimestamperRate(..., start=", str(ts))
 
         # first 30 or so bytes means the first 2 lines
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
+        ts = timestamper.TimestamperRate(input, start, 8000)
         foo = ts.readlines(30)
         eq_(foo, join(lines_out[0:2]))
 
         # stop iteration early
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
+        ts = timestamper.TimestamperRate(input, start, 8000,
                                                 1332561600.000200)
         foo = ""
         for line in ts:
@@ -49,21 +51,21 @@ class TestTimestamper(object):
 
         # stop iteration early (readlines)
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
+        ts = timestamper.TimestamperRate(input, start, 8000,
                                                 1332561600.000200)
         foo = ts.readlines()
         eq_(foo, join(lines_out[0:2]))
 
         # stop iteration really early
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
+        ts = timestamper.TimestamperRate(input, start, 8000,
                                                 1332561600.000000)
         foo = ts.readlines()
         eq_(foo, "")
 
         # use iterator
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
+        ts = timestamper.TimestamperRate(input, start, 8000)
         foo = ""
         for line in ts:
             foo += line
@@ -71,21 +73,21 @@ class TestTimestamper(object):
 
         # check that TimestamperNow gives similar result
         input = cStringIO.StringIO(join(lines_in))
-        ts = nilmdb.timestamper.TimestamperNow(input)
+        ts = timestamper.TimestamperNow(input)
         foo = ts.readlines()
         ne_(foo, join(lines_out))
         eq_(len(foo), len(join(lines_out)))
         eq_(str(ts), "TimestamperNow(...)")
 
         # Test passing a file (should be empty)
-        ts = nilmdb.timestamper.TimestamperNow("/dev/null")
+        ts = timestamper.TimestamperNow("/dev/null")
         for line in ts:
             raise AssertionError
         ts.close()
 
         # Test the null timestamper
         input = cStringIO.StringIO(join(lines_out))  # note: lines_out
-        ts = nilmdb.timestamper.TimestamperNull(input)
+        ts = timestamper.TimestamperNull(input)
         foo = ts.readlines()
         eq_(foo, join(lines_out))
         eq_(str(ts), "TimestamperNull(...)")