Browse Source

Refactor utility classes into nilmdb.utils subdir/namespace

There's some bug with the testing harness where placing e.g.
  from du import du
in nilmdb/utils/__init__.py doesn't quite work -- sometimes the
module "du" replaces the function "du".  Not exactly sure why;
we work around that by just renaming files so they don't match
the imported names directly.
tags/replace-pytables
Jim Paris 9 years ago
parent
commit
f355c73209
34 changed files with 58 additions and 57 deletions
  1. +0
    -7
      nilmdb/__init__.py
  2. +1
    -1
      nilmdb/client.py
  3. +1
    -1
      nilmdb/cmdline/cmdline.py
  4. +1
    -1
      nilmdb/cmdline/create.py
  5. +1
    -1
      nilmdb/cmdline/destroy.py
  6. +1
    -1
      nilmdb/cmdline/extract.py
  7. +1
    -1
      nilmdb/cmdline/info.py
  8. +1
    -1
      nilmdb/cmdline/insert.py
  9. +1
    -1
      nilmdb/cmdline/list.py
  10. +1
    -1
      nilmdb/cmdline/metadata.py
  11. +3
    -4
      nilmdb/httpclient.py
  12. +2
    -3
      nilmdb/nilmdb.py
  13. +3
    -4
      nilmdb/server.py
  14. +1
    -1
      nilmdb/timestamper.py
  15. +7
    -0
      nilmdb/utils/__init__.py
  16. +0
    -0
      nilmdb/utils/diskusage.py
  17. +0
    -0
      nilmdb/utils/iteratorizer.py
  18. +2
    -2
      nilmdb/utils/lrucache.py
  19. +0
    -0
      nilmdb/utils/printf.py
  20. +3
    -0
      nilmdb/utils/serializer.py
  21. +0
    -0
      nilmdb/utils/timer.py
  22. +1
    -0
      setup.cfg
  23. +1
    -1
      tests/renderdot.py
  24. +1
    -1
      tests/test_client.py
  25. +1
    -1
      tests/test_cmdline.py
  26. +1
    -1
      tests/test_interval.py
  27. +10
    -9
      tests/test_iteratorizer.py
  28. +1
    -1
      tests/test_layout.py
  29. +4
    -6
      tests/test_lrucache.py
  30. +3
    -2
      tests/test_nilmdb.py
  31. +1
    -1
      tests/test_printf.py
  32. +1
    -1
      tests/test_rbtree.py
  33. +2
    -2
      tests/test_serializer.py
  34. +1
    -1
      tests/test_timestamper.py

+ 0
- 7
nilmdb/__init__.py View File

@@ -3,14 +3,7 @@
from .nilmdb import NilmDB
from .server import Server
from .client import Client
from .timer import Timer

import cmdline

import pyximport; pyximport.install()
import layout

import serializer
import timestamper
import interval
import du

+ 1
- 1
nilmdb/client.py View File

@@ -3,7 +3,7 @@
"""Class for performing HTTP client requests via libcurl"""

from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *

import time
import sys


+ 1
- 1
nilmdb/cmdline/cmdline.py View File

@@ -1,7 +1,7 @@
"""Command line client functionality"""

from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client

import datetime_tz


+ 1
- 1
nilmdb/cmdline/create.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client

from argparse import ArgumentDefaultsHelpFormatter as def_form


+ 1
- 1
nilmdb/cmdline/destroy.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client

from argparse import ArgumentDefaultsHelpFormatter as def_form


+ 1
- 1
nilmdb/cmdline/extract.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client
import sys



+ 1
- 1
nilmdb/cmdline/info.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *

from argparse import ArgumentDefaultsHelpFormatter as def_form



+ 1
- 1
nilmdb/cmdline/insert.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client
import nilmdb.timestamper



+ 1
- 1
nilmdb/cmdline/list.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client

import fnmatch


+ 1
- 1
nilmdb/cmdline/metadata.py View File

@@ -1,5 +1,5 @@
from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.client

def setup(self, sub):


+ 3
- 4
nilmdb/httpclient.py View File

@@ -1,7 +1,8 @@
"""HTTP client library"""

from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.utils

import time
import sys
@@ -13,8 +14,6 @@ import urllib
import pycurl
import cStringIO

import nilmdb.iteratorizer

class Error(Exception):
"""Base exception for both ClientError and ServerError responses"""
def __init__(self,
@@ -109,7 +108,7 @@ class HTTPClient(object):
self.curl.setopt(pycurl.WRITEFUNCTION, callback)
self.curl.perform()
try:
for i in nilmdb.iteratorizer.Iteratorizer(func):
for i in nilmdb.utils.Iteratorizer(func):
if self._status == 200:
# If we had a 200 response, yield the data to the caller.
yield i


+ 2
- 3
nilmdb/nilmdb.py View File

@@ -11,7 +11,7 @@ Manages both the SQL database and the PyTables storage backend.
# but will pull the nilmdb module instead.
from __future__ import absolute_import
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *

import sqlite3
import tables
@@ -24,7 +24,6 @@ import bisect
import pyximport
pyximport.install()
from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
from nilmdb.lrucache import lrucache

# Note about performance and transactions:
#
@@ -169,7 +168,7 @@ class NilmDB(object):
with self.con:
cur.execute("PRAGMA user_version = {v:d}".format(v=version))

@lrucache(size = 16)
@nilmdb.utils.lru_cache(size = 16)
def _get_intervals(self, stream_id):
"""
Return a mutable IntervalSet corresponding to the given stream ID.


+ 3
- 4
nilmdb/server.py View File

@@ -3,10 +3,9 @@
# Need absolute_import so that "import nilmdb" won't pull in nilmdb.py,
# but will pull the nilmdb module instead.
from __future__ import absolute_import
from nilmdb.utils.printf import *
import nilmdb

from nilmdb.printf import *

import cherrypy
import sys
import time
@@ -59,7 +58,7 @@ class Root(NilmApp):
@cherrypy.expose
@cherrypy.tools.json_out()
def dbsize(self):
return nilmdb.du.du(self.db.get_basepath())
return nilmdb.utils.du(self.db.get_basepath())

class Stream(NilmApp):
"""Stream-specific operations"""
@@ -323,7 +322,7 @@ class Server(object):
# Need to wrap DB object in a serializer because we'll call
# into it from separate threads.
self.embedded = embedded
self.db = nilmdb.serializer.WrapObject(db)
self.db = nilmdb.utils.Serializer(db)
cherrypy.config.update({
'server.socket_host': host,
'server.socket_port': port,


+ 1
- 1
nilmdb/timestamper.py View File

@@ -1,7 +1,7 @@
"""File-like objects that add timestamps to the input lines"""

from __future__ import absolute_import
from nilmdb.printf import *
from nilmdb.utils.printf import *

import time
import os


+ 7
- 0
nilmdb/utils/__init__.py View File

@@ -0,0 +1,7 @@
"""NilmDB utilities"""

from .timer import Timer
from .iteratorizer import Iteratorizer
from .serializer import Serializer
from .lrucache import lru_cache
from .diskusage import du

nilmdb/du.py → nilmdb/utils/diskusage.py View File


nilmdb/iteratorizer.py → nilmdb/utils/iteratorizer.py View File


nilmdb/lrucache.py → nilmdb/utils/lrucache.py View File

@@ -6,10 +6,10 @@
import collections
import functools

def lrucache(size = 10, onremove = None):
def lru_cache(size = 10, onremove = None):
"""Least-recently-used cache decorator.

@lrucache(size = 10, onevict = None)
@lru_cache(size = 10, onevict = None)
def f(...):
pass


nilmdb/printf.py → nilmdb/utils/printf.py View File


nilmdb/serializer.py → nilmdb/utils/serializer.py View File

@@ -67,3 +67,6 @@ class WrapObject(object):
def __del__(self):
self.__wrap_call_queue.put((None, None, None, None))
self.__wrap_serializer.join()

# Just an alias
Serializer = WrapObject

nilmdb/timer.py → nilmdb/utils/timer.py View File


+ 1
- 0
setup.cfg View File

@@ -10,6 +10,7 @@ cover-erase=
##cover-branches= # need nose 1.1.3 for this
stop=
verbosity=2
#tests=tests/test_lrucache.py
#tests=tests/test_cmdline.py
#tests=tests/test_layout.py
#tests=tests/test_rbtree.py


+ 1
- 1
tests/renderdot.py View File

@@ -13,7 +13,7 @@ class Renderer(object):

# Rendering
def __render_dot_node(self, node, max_depth = 20):
from nilmdb.printf import sprintf
from nilmdb.utils.printf import sprintf
"""Render a single node and its children into a dot graph fragment"""
if max_depth == 0:
return ""


+ 1
- 1
tests/test_client.py View File

@@ -1,5 +1,5 @@
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *
from nilmdb.client import ClientError, ServerError

import datetime_tz


+ 1
- 1
tests/test_cmdline.py View File

@@ -1,5 +1,5 @@
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *
import nilmdb.cmdline

from nose.tools import *


+ 1
- 1
tests/test_interval.py View File

@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-

import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *
import datetime_tz

from nose.tools import *


+ 10
- 9
tests/test_iteratorizer.py View File

@@ -1,5 +1,5 @@
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *

import nose
from nose.tools import *
@@ -9,8 +9,6 @@ import time

from test_helpers import *

import nilmdb.iteratorizer

def func_with_callback(a, b, callback):
callback(a)
callback(b)
@@ -27,16 +25,18 @@ class TestIteratorizer(object):
eq_(self.result, "123")

# Now make it an iterator
it = nilmdb.iteratorizer.Iteratorizer(lambda x:
func_with_callback(1, 2, x))
it = nilmdb.utils.Iteratorizer(
lambda x:
func_with_callback(1, 2, x))
result = ""
for i in it:
result += str(i)
eq_(result, "123")

# Make sure things work when an exception occurs
it = nilmdb.iteratorizer.Iteratorizer(lambda x:
func_with_callback(1, "a", x))
it = nilmdb.utils.Iteratorizer(
lambda x:
func_with_callback(1, "a", x))
result = ""
with assert_raises(TypeError) as e:
for i in it:
@@ -48,7 +48,8 @@ class TestIteratorizer(object):
# itself. This doesn't have a particular result in the test,
# but gains coverage.
def foo():
it = nilmdb.iteratorizer.Iteratorizer(lambda x:
func_with_callback(1, 2, x))
it = nilmdb.utils.Iteratorizer(
lambda x:
func_with_callback(1, 2, x))
it.next()
foo()

+ 1
- 1
tests/test_layout.py View File

@@ -2,7 +2,7 @@

import nilmdb

from nilmdb.printf import *
from nilmdb.utils.printf import *

from nose.tools import *
from nose.tools import assert_raises


+ 4
- 6
tests/test_lrucache.py View File

@@ -1,5 +1,5 @@
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *

import nose
from nose.tools import *
@@ -9,20 +9,18 @@ import time

from test_helpers import *

from nilmdb.lrucache import lrucache

@lrucache(size = 3)
@nilmdb.utils.lru_cache(size = 3)
def foo1(n):
return n

@lrucache(size = 5)
@nilmdb.utils.lru_cache(size = 5)
def foo2(n):
return n

def foo3d(n):
foo3d.destructed.append(n)
foo3d.destructed = []
@lrucache(size = 3, onremove = foo3d)
@nilmdb.utils.lru_cache(size = 3, onremove = foo3d)
def foo3(n):
return n



+ 3
- 2
tests/test_nilmdb.py View File

@@ -14,6 +14,7 @@ import urllib2
from urllib2 import urlopen, HTTPError
import Queue
import cStringIO
import time

testdb = "tests/testdb"

@@ -39,8 +40,8 @@ class Test00Nilmdb(object): # named 00 so it runs first
capture = cStringIO.StringIO()
old = sys.stdout
sys.stdout = capture
with nilmdb.Timer("test"):
nilmdb.timer.time.sleep(0.01)
with nilmdb.utils.Timer("test"):
time.sleep(0.01)
sys.stdout = old
in_("test: ", capture.getvalue())



+ 1
- 1
tests/test_printf.py View File

@@ -1,5 +1,5 @@
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *

from nose.tools import *
from nose.tools import assert_raises


+ 1
- 1
tests/test_rbtree.py View File

@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-

import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *

from nose.tools import *
from nose.tools import assert_raises


+ 2
- 2
tests/test_serializer.py View File

@@ -1,5 +1,5 @@
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *

import nose
from nose.tools import *
@@ -57,7 +57,7 @@ class TestUnserialized(Base):
class TestSerialized(Base):
def setUp(self):
self.realfoo = Foo()
self.foo = nilmdb.serializer.WrapObject(self.realfoo)
self.foo = nilmdb.utils.Serializer(self.realfoo)

def tearDown(self):
del self.foo


+ 1
- 1
tests/test_timestamper.py View File

@@ -1,5 +1,5 @@
import nilmdb
from nilmdb.printf import *
from nilmdb.utils.printf import *

import datetime_tz



Loading…
Cancel
Save