Browse Source

Refactor utility classes into nilmdb.utils subdir/namespace

There's some bug with the testing harness where placing e.g.
  from du import du
in nilmdb/utils/__init__.py doesn't quite work -- sometimes the
module "du" replaces the function "du".  Not exactly sure why;
we work around that by just renaming files so they don't match
the imported names directly.
tags/replace-pytables
Jim Paris 11 years ago
parent
commit
f355c73209
34 changed files with 58 additions and 57 deletions
  1. +0
    -7
      nilmdb/__init__.py
  2. +1
    -1
      nilmdb/client.py
  3. +1
    -1
      nilmdb/cmdline/cmdline.py
  4. +1
    -1
      nilmdb/cmdline/create.py
  5. +1
    -1
      nilmdb/cmdline/destroy.py
  6. +1
    -1
      nilmdb/cmdline/extract.py
  7. +1
    -1
      nilmdb/cmdline/info.py
  8. +1
    -1
      nilmdb/cmdline/insert.py
  9. +1
    -1
      nilmdb/cmdline/list.py
  10. +1
    -1
      nilmdb/cmdline/metadata.py
  11. +3
    -4
      nilmdb/httpclient.py
  12. +2
    -3
      nilmdb/nilmdb.py
  13. +3
    -4
      nilmdb/server.py
  14. +1
    -1
      nilmdb/timestamper.py
  15. +7
    -0
      nilmdb/utils/__init__.py
  16. +0
    -0
      nilmdb/utils/diskusage.py
  17. +0
    -0
      nilmdb/utils/iteratorizer.py
  18. +2
    -2
      nilmdb/utils/lrucache.py
  19. +0
    -0
      nilmdb/utils/printf.py
  20. +3
    -0
      nilmdb/utils/serializer.py
  21. +0
    -0
      nilmdb/utils/timer.py
  22. +1
    -0
      setup.cfg
  23. +1
    -1
      tests/renderdot.py
  24. +1
    -1
      tests/test_client.py
  25. +1
    -1
      tests/test_cmdline.py
  26. +1
    -1
      tests/test_interval.py
  27. +10
    -9
      tests/test_iteratorizer.py
  28. +1
    -1
      tests/test_layout.py
  29. +4
    -6
      tests/test_lrucache.py
  30. +3
    -2
      tests/test_nilmdb.py
  31. +1
    -1
      tests/test_printf.py
  32. +1
    -1
      tests/test_rbtree.py
  33. +2
    -2
      tests/test_serializer.py
  34. +1
    -1
      tests/test_timestamper.py

+ 0
- 7
nilmdb/__init__.py View File

@@ -3,14 +3,7 @@
from .nilmdb import NilmDB from .nilmdb import NilmDB
from .server import Server from .server import Server
from .client import Client from .client import Client
from .timer import Timer

import cmdline


import pyximport; pyximport.install() import pyximport; pyximport.install()
import layout import layout

import serializer
import timestamper
import interval import interval
import du

+ 1
- 1
nilmdb/client.py View File

@@ -3,7 +3,7 @@
"""Class for performing HTTP client requests via libcurl""" """Class for performing HTTP client requests via libcurl"""


from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *


import time import time
import sys import sys


+ 1
- 1
nilmdb/cmdline/cmdline.py View File

@@ -1,7 +1,7 @@
"""Command line client functionality""" """Command line client functionality"""


from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client import nilmdb.client


import datetime_tz import datetime_tz


+ 1
- 1
nilmdb/cmdline/create.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client import nilmdb.client


from argparse import ArgumentDefaultsHelpFormatter as def_form from argparse import ArgumentDefaultsHelpFormatter as def_form


+ 1
- 1
nilmdb/cmdline/destroy.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client import nilmdb.client


from argparse import ArgumentDefaultsHelpFormatter as def_form from argparse import ArgumentDefaultsHelpFormatter as def_form


+ 1
- 1
nilmdb/cmdline/extract.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client import nilmdb.client
import sys import sys




+ 1
- 1
nilmdb/cmdline/info.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *


from argparse import ArgumentDefaultsHelpFormatter as def_form from argparse import ArgumentDefaultsHelpFormatter as def_form




+ 1
- 1
nilmdb/cmdline/insert.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client import nilmdb.client
import nilmdb.timestamper import nilmdb.timestamper




+ 1
- 1
nilmdb/cmdline/list.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client import nilmdb.client


import fnmatch import fnmatch


+ 1
- 1
nilmdb/cmdline/metadata.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client import nilmdb.client


def setup(self, sub): def setup(self, sub):


+ 3
- 4
nilmdb/httpclient.py View File

@@ -1,7 +1,8 @@
"""HTTP client library""" """HTTP client library"""


from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.utils


import time import time
import sys import sys
@@ -13,8 +14,6 @@ import urllib
import pycurl import pycurl
import cStringIO import cStringIO


import nilmdb.iteratorizer

class Error(Exception): class Error(Exception):
"""Base exception for both ClientError and ServerError responses""" """Base exception for both ClientError and ServerError responses"""
def __init__(self, def __init__(self,
@@ -109,7 +108,7 @@ class HTTPClient(object):
self.curl.setopt(pycurl.WRITEFUNCTION, callback) self.curl.setopt(pycurl.WRITEFUNCTION, callback)
self.curl.perform() self.curl.perform()
try: try:
for i in nilmdb.iteratorizer.Iteratorizer(func):
for i in nilmdb.utils.Iteratorizer(func):
if self._status == 200: if self._status == 200:
# If we had a 200 response, yield the data to the caller. # If we had a 200 response, yield the data to the caller.
yield i yield i


+ 2
- 3
nilmdb/nilmdb.py View File

@@ -11,7 +11,7 @@ Manages both the SQL database and the PyTables storage backend.
# but will pull the nilmdb module instead. # but will pull the nilmdb module instead.
from __future__ import absolute_import from __future__ import absolute_import
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *


import sqlite3 import sqlite3
import tables import tables
@@ -24,7 +24,6 @@ import bisect
import pyximport import pyximport
pyximport.install() pyximport.install()
from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
from nilmdb.lrucache import lrucache


# Note about performance and transactions: # Note about performance and transactions:
# #
@@ -169,7 +168,7 @@ class NilmDB(object):
with self.con: with self.con:
cur.execute("PRAGMA user_version = {v:d}".format(v=version)) cur.execute("PRAGMA user_version = {v:d}".format(v=version))


@lrucache(size = 16)
@nilmdb.utils.lru_cache(size = 16)
def _get_intervals(self, stream_id): def _get_intervals(self, stream_id):
""" """
Return a mutable IntervalSet corresponding to the given stream ID. Return a mutable IntervalSet corresponding to the given stream ID.


+ 3
- 4
nilmdb/server.py View File

@@ -3,10 +3,9 @@
# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py, # Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
# but will pull the nilmdb module instead. # but will pull the nilmdb module instead.
from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb import nilmdb


from nilmdb.printf import *

import cherrypy import cherrypy
import sys import sys
import time import time
@@ -59,7 +58,7 @@ class Root(NilmApp):
@cherrypy.expose @cherrypy.expose
@cherrypy.tools.json_out() @cherrypy.tools.json_out()
def dbsize(self): def dbsize(self):
return nilmdb.du.du(self.db.get_basepath())
return nilmdb.utils.du(self.db.get_basepath())


class Stream(NilmApp): class Stream(NilmApp):
"""Stream-specific operations""" """Stream-specific operations"""
@@ -323,7 +322,7 @@ class Server(object):
# Need to wrap DB object in a serializer because we'll call # Need to wrap DB object in a serializer because we'll call
# into it from separate threads. # into it from separate threads.
self.embedded = embedded self.embedded = embedded
self.db = nilmdb.serializer.WrapObject(db)
self.db = nilmdb.utils.Serializer(db)
cherrypy.config.update({ cherrypy.config.update({
'server.socket_host': host, 'server.socket_host': host,
'server.socket_port': port, 'server.socket_port': port,


+ 1
- 1
nilmdb/timestamper.py View File

@@ -1,7 +1,7 @@
"""File-like objects that add timestamps to the input lines""" """File-like objects that add timestamps to the input lines"""


from __future__ import absolute_import from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *


import time import time
import os import os


+ 7
- 0
nilmdb/utils/__init__.py View File

@@ -0,0 +1,7 @@
"""NilmDB utilities"""

from .timer import Timer
from .iteratorizer import Iteratorizer
from .serializer import Serializer
from .lrucache import lru_cache
from .diskusage import du

nilmdb/du.py → nilmdb/utils/diskusage.py View File


nilmdb/iteratorizer.py → nilmdb/utils/iteratorizer.py View File


nilmdb/lrucache.py → nilmdb/utils/lrucache.py View File

@@ -6,10 +6,10 @@
import collections import collections
import functools import functools


def lrucache(size = 10, onremove = None):
def lru_cache(size = 10, onremove = None):
"""Least-recently-used cache decorator. """Least-recently-used cache decorator.


@lrucache(size = 10, onevict = None)
@lru_cache(size = 10, onevict = None)
def f(...): def f(...):
pass pass



nilmdb/printf.py → nilmdb/utils/printf.py View File


nilmdb/serializer.py → nilmdb/utils/serializer.py View File

@@ -67,3 +67,6 @@ class WrapObject(object):
def __del__(self): def __del__(self):
self.__wrap_call_queue.put((None, None, None, None)) self.__wrap_call_queue.put((None, None, None, None))
self.__wrap_serializer.join() self.__wrap_serializer.join()

# Just an alias
Serializer = WrapObject

nilmdb/timer.py → nilmdb/utils/timer.py View File


+ 1
- 0
setup.cfg View File

@@ -10,6 +10,7 @@ cover-erase=
##cover-branches= # need nose 1.1.3 for this ##cover-branches= # need nose 1.1.3 for this
stop= stop=
verbosity=2 verbosity=2
#tests=tests/test_lrucache.py
#tests=tests/test_cmdline.py #tests=tests/test_cmdline.py
#tests=tests/test_layout.py #tests=tests/test_layout.py
#tests=tests/test_rbtree.py #tests=tests/test_rbtree.py


+ 1
- 1
tests/renderdot.py View File

@@ -13,7 +13,7 @@ class Renderer(object):


# Rendering # Rendering
def __render_dot_node(self, node, max_depth = 20): def __render_dot_node(self, node, max_depth = 20):
from nilmdb.printf import sprintf
from nilmdb.utils.printf import sprintf
"""Render a single node and its children into a dot graph fragment""" """Render a single node and its children into a dot graph fragment"""
if max_depth == 0: if max_depth == 0:
return "" return ""


+ 1
- 1
tests/test_client.py View File

@@ -1,5 +1,5 @@
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *
from nilmdb.client import ClientError, ServerError from nilmdb.client import ClientError, ServerError


import datetime_tz import datetime_tz


+ 1
- 1
tests/test_cmdline.py View File

@@ -1,5 +1,5 @@
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.cmdline import nilmdb.cmdline


from nose.tools import * from nose.tools import *


+ 1
- 1
tests/test_interval.py View File

@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-


import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *
import datetime_tz import datetime_tz


from nose.tools import * from nose.tools import *


+ 10
- 9
tests/test_iteratorizer.py View File

@@ -1,5 +1,5 @@
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *


import nose import nose
from nose.tools import * from nose.tools import *
@@ -9,8 +9,6 @@ import time


from test_helpers import * from test_helpers import *


import nilmdb.iteratorizer

def func_with_callback(a, b, callback): def func_with_callback(a, b, callback):
callback(a) callback(a)
callback(b) callback(b)
@@ -27,16 +25,18 @@ class TestIteratorizer(object):
eq_(self.result, "123") eq_(self.result, "123")


# Now make it an iterator # Now make it an iterator
it = nilmdb.iteratorizer.Iteratorizer(lambda x:
func_with_callback(1, 2, x))
it = nilmdb.utils.Iteratorizer(
lambda x:
func_with_callback(1, 2, x))
result = "" result = ""
for i in it: for i in it:
result += str(i) result += str(i)
eq_(result, "123") eq_(result, "123")


# Make sure things work when an exception occurs # Make sure things work when an exception occurs
it = nilmdb.iteratorizer.Iteratorizer(lambda x:
func_with_callback(1, "a", x))
it = nilmdb.utils.Iteratorizer(
lambda x:
func_with_callback(1, "a", x))
result = "" result = ""
with assert_raises(TypeError) as e: with assert_raises(TypeError) as e:
for i in it: for i in it:
@@ -48,7 +48,8 @@ class TestIteratorizer(object):
# itself. This doesn't have a particular result in the test, # itself. This doesn't have a particular result in the test,
# but gains coverage. # but gains coverage.
def foo(): def foo():
it = nilmdb.iteratorizer.Iteratorizer(lambda x:
func_with_callback(1, 2, x))
it = nilmdb.utils.Iteratorizer(
lambda x:
func_with_callback(1, 2, x))
it.next() it.next()
foo() foo()

+ 1
- 1
tests/test_layout.py View File

@@ -2,7 +2,7 @@


import nilmdb import nilmdb


from nilmdb.printf import *
from nilmdb.utils.printf import *


from nose.tools import * from nose.tools import *
from nose.tools import assert_raises from nose.tools import assert_raises


+ 4
- 6
tests/test_lrucache.py View File

@@ -1,5 +1,5 @@
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *


import nose import nose
from nose.tools import * from nose.tools import *
@@ -9,20 +9,18 @@ import time


from test_helpers import * from test_helpers import *


from nilmdb.lrucache import lrucache

@lrucache(size = 3)
@nilmdb.utils.lru_cache(size = 3)
def foo1(n): def foo1(n):
return n return n


@lrucache(size = 5)
@nilmdb.utils.lru_cache(size = 5)
def foo2(n): def foo2(n):
return n return n


def foo3d(n): def foo3d(n):
foo3d.destructed.append(n) foo3d.destructed.append(n)
foo3d.destructed = [] foo3d.destructed = []
@lrucache(size = 3, onremove = foo3d)
@nilmdb.utils.lru_cache(size = 3, onremove = foo3d)
def foo3(n): def foo3(n):
return n return n




+ 3
- 2
tests/test_nilmdb.py View File

@@ -14,6 +14,7 @@ import urllib2
from urllib2 import urlopen, HTTPError from urllib2 import urlopen, HTTPError
import Queue import Queue
import cStringIO import cStringIO
import time


testdb = "tests/testdb" testdb = "tests/testdb"


@@ -39,8 +40,8 @@ class Test00Nilmdb(object): # named 00 so it runs first
capture = cStringIO.StringIO() capture = cStringIO.StringIO()
old = sys.stdout old = sys.stdout
sys.stdout = capture sys.stdout = capture
with nilmdb.Timer("test"):
nilmdb.timer.time.sleep(0.01)
with nilmdb.utils.Timer("test"):
time.sleep(0.01)
sys.stdout = old sys.stdout = old
in_("test: ", capture.getvalue()) in_("test: ", capture.getvalue())




+ 1
- 1
tests/test_printf.py View File

@@ -1,5 +1,5 @@
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *


from nose.tools import * from nose.tools import *
from nose.tools import assert_raises from nose.tools import assert_raises


+ 1
- 1
tests/test_rbtree.py View File

@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-


import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *


from nose.tools import * from nose.tools import *
from nose.tools import assert_raises from nose.tools import assert_raises


+ 2
- 2
tests/test_serializer.py View File

@@ -1,5 +1,5 @@
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *


import nose import nose
from nose.tools import * from nose.tools import *
@@ -57,7 +57,7 @@ class TestUnserialized(Base):
class TestSerialized(Base): class TestSerialized(Base):
def setUp(self): def setUp(self):
self.realfoo = Foo() self.realfoo = Foo()
self.foo = nilmdb.serializer.WrapObject(self.realfoo)
self.foo = nilmdb.utils.Serializer(self.realfoo)


def tearDown(self): def tearDown(self):
del self.foo del self.foo


+ 1
- 1
tests/test_timestamper.py View File

@@ -1,5 +1,5 @@
import nilmdb import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *


import datetime_tz import datetime_tz




Loading…
Cancel
Save