Browse Source

Huge cleanup of namespaces, modules, packages, imports.

Now nilmdb.client, nilmdb.server, nilmdb.cmdline, and nilmdb.utils
are each their own modules, and there is a little bit more of a
logical separation between them.  Various changes scattered throughout
to fix naming (for example, nilmdb.nilmdb.NilmDBError is now
nilmdb.server.errors.NilmDBError).

Reduced usage of "from __future__ import absolute_import" as much
as possible.  It's still needed for the functions in the nilmdb/server
directory to be able to import the nilmdb module rather than the
nilmdb.py script.

This should hopefully ease future packaging a bit.
tags/nilmdb-0.2^2
Jim Paris 8 years ago
parent
commit
225003f412
35 changed files with 131 additions and 127 deletions
  1. +2
    -10
      nilmdb/__init__.py
  2. +4
    -0
      nilmdb/client/__init__.py
  3. +4
    -8
      nilmdb/client/client.py
  4. +33
    -0
      nilmdb/client/errors.py
  5. +3
    -35
      nilmdb/client/httpclient.py
  6. +2
    -0
      nilmdb/cmdline/__init__.py
  7. +1
    -2
      nilmdb/cmdline/cmdline.py
  8. +1
    -1
      nilmdb/cmdline/create.py
  9. +1
    -1
      nilmdb/cmdline/destroy.py
  10. +1
    -1
      nilmdb/cmdline/extract.py
  11. +0
    -1
      nilmdb/cmdline/info.py
  12. +4
    -4
      nilmdb/cmdline/insert.py
  13. +1
    -1
      nilmdb/cmdline/list.py
  14. +1
    -1
      nilmdb/cmdline/metadata.py
  15. +1
    -2
      nilmdb/cmdline/remove.py
  16. +14
    -0
      nilmdb/server/__init__.py
  17. +3
    -1
      nilmdb/server/bulkdata.py
  18. +12
    -0
      nilmdb/server/errors.py
  19. +0
    -0
      nilmdb/server/interval.pyx
  20. +0
    -0
      nilmdb/server/interval.pyxdep
  21. +0
    -0
      nilmdb/server/layout.pyx
  22. +6
    -19
      nilmdb/server/nilmdb.py
  23. +0
    -0
      nilmdb/server/rbtree.pxd
  24. +0
    -0
      nilmdb/server/rbtree.pyx
  25. +0
    -0
      nilmdb/server/rbtree.pyxdep
  26. +9
    -10
      nilmdb/server/server.py
  27. +0
    -1
      nilmdb/utils/diskusage.py
  28. +0
    -1
      nilmdb/utils/timestamper.py
  29. +0
    -3
      nilmdb/utils/urllib.py
  30. +2
    -3
      tests/test_bulkdata.py
  31. +8
    -7
      tests/test_client.py
  32. +2
    -1
      tests/test_interval.py
  33. +4
    -4
      tests/test_layout.py
  34. +1
    -1
      tests/test_rbtree.py
  35. +11
    -9
      tests/test_timestamper.py

+ 2
- 10
nilmdb/__init__.py View File

@@ -1,12 +1,4 @@
"""Main NilmDB import"""

from .nilmdb import NilmDB
from .server import Server
from .client import Client

import pyximport; pyximport.install()
import layout
import interval

import cmdline

from server import NilmDB, Server
from client import Client

+ 4
- 0
nilmdb/client/__init__.py View File

@@ -0,0 +1,4 @@
"""nilmdb.client"""

from .client import Client
from .errors import *

nilmdb/client.py → nilmdb/client/client.py View File

@@ -2,7 +2,9 @@

"""Class for performing HTTP client requests via libcurl"""

from __future__ import absolute_import
import nilmdb
import nilmdb.utils
import nilmdb.client.httpclient
from nilmdb.utils.printf import *

import time
@@ -12,12 +14,6 @@ import os
import simplejson as json
import itertools

import nilmdb.utils
import nilmdb.httpclient

# Other functions expect to see these in the nilmdb.client namespace
from nilmdb.httpclient import ClientError, ServerError, Error

version = "1.0"

def float_to_string(f):
@@ -30,7 +26,7 @@ class Client(object):
client_version = version

def __init__(self, url):
self.http = nilmdb.httpclient.HTTPClient(url)
self.http = nilmdb.client.httpclient.HTTPClient(url)

def _json_param(self, data):
"""Return compact json-encoded version of parameter"""

+ 33
- 0
nilmdb/client/errors.py View File

@@ -0,0 +1,33 @@
"""HTTP client errors"""

from nilmdb.utils.printf import *

class Error(Exception):
"""Base exception for both ClientError and ServerError responses"""
def __init__(self,
status = "Unspecified error",
message = None,
url = None,
traceback = None):
Exception.__init__(self, status)
self.status = status # e.g. "400 Bad Request"
self.message = message # textual message from the server
self.url = url # URL we were requesting
self.traceback = traceback # server traceback, if available
def _format_error(self, show_url):
s = sprintf("[%s]", self.status)
if self.message:
s += sprintf(" %s", self.message)
if show_url and self.url: # pragma: no cover
s += sprintf(" (%s)", self.url)
if self.traceback: # pragma: no cover
s += sprintf("\nServer traceback:\n%s", self.traceback)
return s
def __str__(self):
return self._format_error(show_url = False)
def __repr__(self): # pragma: no cover
return self._format_error(show_url = True)
class ClientError(Error):
pass
class ServerError(Error):
pass

nilmdb/httpclient.py → nilmdb/client/httpclient.py View File

@@ -1,8 +1,9 @@
"""HTTP client library"""

from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.utils
from nilmdb.utils.printf import *
from nilmdb.client.errors import *

import time
import sys
@@ -13,39 +14,6 @@ import urlparse
import pycurl
import cStringIO

class Error(Exception):
"""Base exception for both ClientError and ServerError responses"""
def __init__(self,
status = "Unspecified error",
message = None,
url = None,
traceback = None):
Exception.__init__(self, status)
self.status = status # e.g. "400 Bad Request"
self.message = message # textual message from the server
self.url = url # URL we were requesting
self.traceback = traceback # server traceback, if available
def __str__(self):
s = sprintf("[%s]", self.status)
if self.message:
s += sprintf(" %s", self.message)
if self.traceback: # pragma: no cover
s += sprintf("\nServer traceback:\n%s", self.traceback)
return s
def __repr__(self): # pragma: no cover
s = sprintf("[%s]", self.status)
if self.message:
s += sprintf(" %s", self.message)
if self.url:
s += sprintf(" (%s)", self.url)
if self.traceback:
s += sprintf("\nServer traceback:\n%s", self.traceback)
return s
class ClientError(Error):
pass
class ServerError(Error):
pass

class HTTPClient(object):
"""Class to manage and perform HTTP requests from the client"""
def __init__(self, baseurl = ""):

+ 2
- 0
nilmdb/cmdline/__init__.py View File

@@ -1 +1,3 @@
"""nilmdb.cmdline"""

from .cmdline import Cmdline

+ 1
- 2
nilmdb/cmdline/cmdline.py View File

@@ -1,8 +1,7 @@
"""Command line client functionality"""

from __future__ import absolute_import
import nilmdb
from nilmdb.utils.printf import *
import nilmdb.client

import datetime_tz
import dateutil.parser


+ 1
- 1
nilmdb/cmdline/create.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client
import textwrap



+ 1
- 1
nilmdb/cmdline/destroy.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client

from argparse import ArgumentDefaultsHelpFormatter as def_form


+ 1
- 1
nilmdb/cmdline/extract.py View File

@@ -1,6 +1,6 @@
from __future__ import absolute_import
from __future__ import print_function
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client
import sys



+ 0
- 1
nilmdb/cmdline/info.py View File

@@ -1,4 +1,3 @@
from __future__ import absolute_import
from nilmdb.utils.printf import *

from argparse import ArgumentDefaultsHelpFormatter as def_form


+ 4
- 4
nilmdb/cmdline/insert.py View File

@@ -1,7 +1,7 @@
from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client
import nilmdb.timestamper
import nilmdb.utils.timestamper as timestamper

import sys

@@ -69,7 +69,7 @@ def cmd_insert(self):

# Build a timestamper for this file
if self.args.none:
ts = nilmdb.timestamper.TimestamperNull(infile)
ts = timestamper.TimestamperNull(infile)
else:
if self.args.start:
start = self.args.start
@@ -84,7 +84,7 @@ def cmd_insert(self):
self.die("error: --rate is needed, but was not specified")
rate = self.args.rate

ts = nilmdb.timestamper.TimestamperRate(infile, start, rate)
ts = timestamper.TimestamperRate(infile, start, rate)

# Print info
if not self.args.quiet:


+ 1
- 1
nilmdb/cmdline/list.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client

import fnmatch


+ 1
- 1
nilmdb/cmdline/metadata.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client

def setup(self, sub):


+ 1
- 2
nilmdb/cmdline/remove.py View File

@@ -1,6 +1,5 @@
from __future__ import absolute_import
from __future__ import print_function
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client
import sys



+ 14
- 0
nilmdb/server/__init__.py View File

@@ -0,0 +1,14 @@
"""nilmdb.server"""

# Try to set up pyximport to automatically rebuild Cython modules. If
# this doesn't work, it's OK, as long as the modules were built externally.
try:
import pyximport
pyximport.install()
import layout
except: # pragma: no cover
pass

from .nilmdb import NilmDB
from .server import Server
from .errors import *

nilmdb/bulkdata.py → nilmdb/server/bulkdata.py View File

@@ -1,5 +1,7 @@
# Fixed record size bulk data storage

# Need absolute_import so that "import nilmdb" won't pull in
# nilmdb.py, but will pull the parent nilmdb module instead.
from __future__ import absolute_import
from __future__ import division
import nilmdb
@@ -75,7 +77,7 @@ class BulkData(object):

# Get layout, and build format string for struct module
try:
layout = nilmdb.layout.get_named(layout_name)
layout = nilmdb.server.layout.get_named(layout_name)
struct_fmt = '<d' # Little endian, double timestamp
struct_mapping = {
"int8": 'b',

+ 12
- 0
nilmdb/server/errors.py View File

@@ -0,0 +1,12 @@
"""Exceptions"""

class NilmDBError(Exception):
"""Base exception for NilmDB errors"""
def __init__(self, message = "Unspecified error"):
Exception.__init__(self, message)

class StreamError(NilmDBError):
pass

class OverlapError(NilmDBError):
pass

nilmdb/interval.pyx → nilmdb/server/interval.pyx View File


nilmdb/interval.pyxdep → nilmdb/server/interval.pyxdep View File


nilmdb/layout.pyx → nilmdb/server/layout.pyx View File


nilmdb/nilmdb.py → nilmdb/server/nilmdb.py View File

@@ -7,11 +7,15 @@ Object that represents a NILM database file.
Manages both the SQL database and the table storage backend.
"""

# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
# but will pull the nilmdb module instead.
# Need absolute_import so that "import nilmdb" won't pull in
# nilmdb.py, but will pull the parent nilmdb module instead.
from __future__ import absolute_import
import nilmdb
from nilmdb.utils.printf import *
from nilmdb.server.interval import (Interval, DBInterval,
IntervalSet, IntervalError)
from nilmdb.server import bulkdata
from nilmdb.server.errors import *

import sqlite3
import time
@@ -20,12 +24,6 @@ import os
import errno
import bisect

import pyximport
pyximport.install()
from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError

from . import bulkdata

# Note about performance and transactions:
#
# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
@@ -77,17 +75,6 @@ _sql_schema_updates = {
""",
}

class NilmDBError(Exception):
"""Base exception for NilmDB errors"""
def __init__(self, message = "Unspecified error"):
Exception.__init__(self, message)

class StreamError(NilmDBError):
pass

class OverlapError(NilmDBError):
pass

@nilmdb.utils.must_close()
class NilmDB(object):
verbose = 0

nilmdb/rbtree.pxd → nilmdb/server/rbtree.pxd View File


nilmdb/rbtree.pyx → nilmdb/server/rbtree.pyx View File


nilmdb/rbtree.pyxdep → nilmdb/server/rbtree.pyxdep View File


nilmdb/server.py → nilmdb/server/server.py View File

@@ -1,10 +1,11 @@
"""CherryPy-based server for accessing NILM database via HTTP"""

# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
# but will pull the nilmdb module instead.
# Need absolute_import so that "import nilmdb" won't pull in
# nilmdb.py, but will pull the nilmdb module instead.
from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb
from nilmdb.utils.printf import *
from nilmdb.server.errors import *

import cherrypy
import sys
@@ -14,8 +15,6 @@ import simplejson as json
import decorator
import traceback

from nilmdb.nilmdb import NilmDBError

try:
import cherrypy
cherrypy.tools.json_out
@@ -159,7 +158,7 @@ class Stream(NilmApp):
matching the given keys."""
try:
data = self.db.stream_get_metadata(path)
except nilmdb.nilmdb.StreamError as e:
except nilmdb.server.nilmdb.StreamError as e:
raise cherrypy.HTTPError("404 Not Found", e.message)
if key is None: # If no keys specified, return them all
key = data.keys()
@@ -222,9 +221,9 @@ class Stream(NilmApp):

# Parse the input data
try:
parser = nilmdb.layout.Parser(layout)
parser = nilmdb.server.layout.Parser(layout)
parser.parse(body)
except nilmdb.layout.ParserError as e:
except nilmdb.server.layout.ParserError as e:
raise cherrypy.HTTPError("400 Bad Request",
"error parsing input data: " +
e.message)
@@ -249,7 +248,7 @@ class Stream(NilmApp):
# Now do the nilmdb insert, passing it the parser full of data.
try:
result = self.db.stream_insert(path, start, end, parser.data)
except nilmdb.nilmdb.NilmDBError as e:
except NilmDBError as e:
raise cherrypy.HTTPError("400 Bad Request", e.message)

# Done
@@ -349,7 +348,7 @@ class Stream(NilmApp):
layout = streams[0][1]

# Get formatter
formatter = nilmdb.layout.Formatter(layout)
formatter = nilmdb.server.layout.Formatter(layout)

@workaround_cp_bug_1200
def content(start, end, count):

+ 0
- 1
nilmdb/utils/diskusage.py View File

@@ -1,4 +1,3 @@
import nilmdb
import os
from math import log



nilmdb/timestamper.py → nilmdb/utils/timestamper.py View File

@@ -1,6 +1,5 @@
"""File-like objects that add timestamps to the input lines"""

from __future__ import absolute_import
from nilmdb.utils.printf import *

import time

+ 0
- 3
nilmdb/utils/urllib.py View File

@@ -20,9 +20,6 @@ def urlencode(query):
v = quote_plus(v)
l.append(k + '=' + v)
elif _is_unicode(v):
# is there a reasonable way to convert to ASCII?
# encode generates a string, but "replace" or "ignore"
# lose information and "strict" can raise UnicodeError
v = quote_plus(v.encode("utf-8","strict"))
l.append(k + '=' + v)
else:


+ 2
- 3
tests/test_bulkdata.py View File

@@ -2,8 +2,6 @@

import nilmdb
from nilmdb.utils.printf import *
import nilmdb.bulkdata

from nose.tools import *
from nose.tools import assert_raises
import itertools
@@ -12,7 +10,8 @@ from testutil.helpers import *

testdb = "tests/bulkdata-testdb"

from nilmdb.bulkdata import BulkData
import nilmdb.server.bulkdata
from nilmdb.server.bulkdata import BulkData

class TestBulkData(object):



+ 8
- 7
tests/test_client.py View File

@@ -2,6 +2,7 @@

import nilmdb
from nilmdb.utils.printf import *
from nilmdb.utils import timestamper
from nilmdb.client import ClientError, ServerError

import datetime_tz
@@ -158,13 +159,13 @@ class TestClient(object):
rate = 120

# First try a nonexistent path
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
data = timestamper.TimestamperRate(testfile, start, 120)
with assert_raises(ClientError) as e:
result = client.stream_insert("/newton/no-such-path", data)
in_("404 Not Found", str(e.exception))

# Now try reversed timestamps
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
data = timestamper.TimestamperRate(testfile, start, 120)
data = reversed(list(data))
with assert_raises(ClientError) as e:
result = client.stream_insert("/newton/prep", data)
@@ -173,7 +174,7 @@ class TestClient(object):

# Now try empty data (no server request made)
empty = cStringIO.StringIO("")
data = nilmdb.timestamper.TimestamperRate(empty, start, 120)
data = timestamper.TimestamperRate(empty, start, 120)
result = client.stream_insert("/newton/prep", data)
eq_(result, None)

@@ -185,7 +186,7 @@ class TestClient(object):
in_("no data provided", str(e.exception))

# Specify start/end (starts too late)
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
data = timestamper.TimestamperRate(testfile, start, 120)
with assert_raises(ClientError) as e:
result = client.stream_insert("/newton/prep", data,
start + 5, start + 120)
@@ -194,7 +195,7 @@ class TestClient(object):
str(e.exception))

# Specify start/end (ends too early)
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
data = timestamper.TimestamperRate(testfile, start, 120)
with assert_raises(ClientError) as e:
result = client.stream_insert("/newton/prep", data,
start, start + 1)
@@ -205,7 +206,7 @@ class TestClient(object):
str(e.exception))

# Now do the real load
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
data = timestamper.TimestamperRate(testfile, start, 120)
result = client.stream_insert("/newton/prep", data,
start, start + 119.999777)
eq_(result, "ok")
@@ -216,7 +217,7 @@ class TestClient(object):
eq_(intervals, [[start, start + 119.999777]])

# Try some overlapping data -- just insert it again
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
data = timestamper.TimestamperRate(testfile, start, 120)
with assert_raises(ClientError) as e:
result = client.stream_insert("/newton/prep", data)
in_("400 Bad Request", str(e.exception))


+ 2
- 1
tests/test_interval.py View File

@@ -8,7 +8,8 @@ from nose.tools import *
from nose.tools import assert_raises
import itertools

from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
from nilmdb.server.interval import (Interval, DBInterval,
IntervalSet, IntervalError)

from testutil.helpers import *
import unittest


+ 4
- 4
tests/test_layout.py View File

@@ -22,17 +22,17 @@ import unittest

from testutil.helpers import *

from nilmdb.layout import *
from nilmdb.server.layout import *

class TestLayouts(object):
# Some nilmdb.layout tests. Not complete, just fills in missing
# coverage.
def test_layouts(self):
x = nilmdb.layout.get_named("PrepData")
y = nilmdb.layout.get_named("float32_8")
x = nilmdb.server.layout.get_named("PrepData")
y = nilmdb.server.layout.get_named("float32_8")
eq_(x.count, y.count)
eq_(x.datatype, y.datatype)
y = nilmdb.layout.get_named("float32_7")
y = nilmdb.server.layout.get_named("float32_7")
ne_(x.count, y.count)
eq_(x.datatype, y.datatype)



+ 1
- 1
tests/test_rbtree.py View File

@@ -6,7 +6,7 @@ from nilmdb.utils.printf import *
from nose.tools import *
from nose.tools import assert_raises

from nilmdb.rbtree import RBTree, RBNode
from nilmdb.server.rbtree import RBTree, RBNode

from testutil.helpers import *
import unittest


+ 11
- 9
tests/test_timestamper.py View File

@@ -11,6 +11,8 @@ import cStringIO

from testutil.helpers import *

from nilmdb.utils import timestamper

class TestTimestamper(object):

# Not a very comprehensive test, but it's good enough.
@@ -27,20 +29,20 @@ class TestTimestamper(object):

# full
input = cStringIO.StringIO(join(lines_in))
ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
ts = timestamper.TimestamperRate(input, start, 8000)
foo = ts.readlines()
eq_(foo, join(lines_out))
in_("TimestamperRate(..., start=", str(ts))

# first 30 or so bytes means the first 2 lines
input = cStringIO.StringIO(join(lines_in))
ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
ts = timestamper.TimestamperRate(input, start, 8000)
foo = ts.readlines(30)
eq_(foo, join(lines_out[0:2]))

# stop iteration early
input = cStringIO.StringIO(join(lines_in))
ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
ts = timestamper.TimestamperRate(input, start, 8000,
1332561600.000200)
foo = ""
for line in ts:
@@ -49,21 +51,21 @@ class TestTimestamper(object):

# stop iteration early (readlines)
input = cStringIO.StringIO(join(lines_in))
ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
ts = timestamper.TimestamperRate(input, start, 8000,
1332561600.000200)
foo = ts.readlines()
eq_(foo, join(lines_out[0:2]))

# stop iteration really early
input = cStringIO.StringIO(join(lines_in))
ts = nilmdb.timestamper.TimestamperRate(input, start, 8000,
ts = timestamper.TimestamperRate(input, start, 8000,
1332561600.000000)
foo = ts.readlines()
eq_(foo, "")

# use iterator
input = cStringIO.StringIO(join(lines_in))
ts = nilmdb.timestamper.TimestamperRate(input, start, 8000)
ts = timestamper.TimestamperRate(input, start, 8000)
foo = ""
for line in ts:
foo += line
@@ -71,21 +73,21 @@ class TestTimestamper(object):

# check that TimestamperNow gives similar result
input = cStringIO.StringIO(join(lines_in))
ts = nilmdb.timestamper.TimestamperNow(input)
ts = timestamper.TimestamperNow(input)
foo = ts.readlines()
ne_(foo, join(lines_out))
eq_(len(foo), len(join(lines_out)))
eq_(str(ts), "TimestamperNow(...)")

# Test passing a file (should be empty)
ts = nilmdb.timestamper.TimestamperNow("/dev/null")
ts = timestamper.TimestamperNow("/dev/null")
for line in ts:
raise AssertionError
ts.close()

# Test the null timestamper
input = cStringIO.StringIO(join(lines_out)) # note: lines_out
ts = nilmdb.timestamper.TimestamperNull(input)
ts = timestamper.TimestamperNull(input)
foo = ts.readlines()
eq_(foo, join(lines_out))
eq_(str(ts), "TimestamperNull(...)")

Loading…
Cancel
Save