Browse Source

Remove Iteratorizer, as it's no longer needed

tags/nilmdb-1.4.4
Jim Paris 11 years ago
parent
commit
f8ca8d31e6
4 changed files with 0 additions and 163 deletions
  1. +0
    -1
      nilmdb/utils/__init__.py
  2. +0
    -100
      nilmdb/utils/iteratorizer.py
  3. +0
    -1
      tests/test.order
  4. +0
    -61
      tests/test_iteratorizer.py

+ 0
- 1
nilmdb/utils/__init__.py View File

@@ -2,7 +2,6 @@

from __future__ import absolute_import
from nilmdb.utils.timer import Timer
from nilmdb.utils.iteratorizer import Iteratorizer
from nilmdb.utils.serializer import serializer_proxy
from nilmdb.utils.lrucache import lru_cache
from nilmdb.utils.diskusage import du, human_size


+ 0
- 100
nilmdb/utils/iteratorizer.py View File

@@ -1,100 +0,0 @@
import Queue
import threading
import sys
import contextlib

# This file provides a context manager that converts a function
# that takes a callback into a generator that returns an iterable.
# This is done by running the function in a new thread.

# Based partially on http://stackoverflow.com/questions/9968592/

class IteratorizerThread(threading.Thread):
def __init__(self, queue, function, curl_hack):
"""
function: function to execute, which takes the
callback (provided by this class) as an argument
"""
threading.Thread.__init__(self)
self.name = "Iteratorizer-" + function.__name__ + "-" + self.name
self.function = function
self.queue = queue
self.die = False
self.curl_hack = curl_hack

def callback(self, data):
try:
if self.die:
raise Exception() # trigger termination
self.queue.put((1, data))
except:
if self.curl_hack:
# We can't raise exceptions, because the pycurl
# extension module will unconditionally print the
# exception itself, and not pass it up to the caller.
# Instead, just return a value that tells curl to
# abort. (-1 would be best, in case we were given 0
# bytes, but the extension doesn't support that).
self.queue.put((2, sys.exc_info()))
return 0
raise

def run(self):
try:
result = self.function(self.callback)
except:
self.queue.put((2, sys.exc_info()))
else:
self.queue.put((0, result))

@contextlib.contextmanager
def Iteratorizer(function, curl_hack = False):
"""
Context manager that takes a function expecting a callback,
and provides an iterable that yields the values passed to that
callback instead.

function: function to execute, which takes a callback
(provided by this context manager) as an argument

with iteratorizer(func) as it:
for i in it:
print 'callback was passed:', i
print 'function returned:', it.retval
"""
queue = Queue.Queue(maxsize = 1)
thread = IteratorizerThread(queue, function, curl_hack)
thread.daemon = True
thread.start()

class iteratorizer_gen(object):
def __init__(self, queue):
self.queue = queue
self.retval = None

def __iter__(self):
return self

def next(self):
(typ, data) = self.queue.get()
if typ == 0:
# function has returned
self.retval = data
raise StopIteration
elif typ == 1:
# data is available
return data
else:
# callback raised an exception
raise data[0], data[1], data[2]

try:
yield iteratorizer_gen(queue)
finally:
# Ask the thread to die, if it's still running.
thread.die = True
while thread.isAlive():
try:
queue.get(True, 0.01)
except: # pragma: no cover
pass

+ 0
- 1
tests/test.order View File

@@ -4,7 +4,6 @@ test_lrucache.py
test_mustclose.py

test_serializer.py
test_iteratorizer.py

test_timestamper.py
test_rbtree.py


+ 0
- 61
tests/test_iteratorizer.py View File

@@ -1,61 +0,0 @@
import nilmdb
from nilmdb.utils.printf import *

import nose
from nose.tools import *
from nose.tools import assert_raises
import threading
import time

from testutil.helpers import *

def func_with_callback(a, b, callback):
callback(a)
callback(b)
callback(a+b)
return "return value"

class TestIteratorizer(object):
def test(self):

# First try it with a normal callback
self.result = ""
def cb(x):
self.result += str(x)
func_with_callback(1, 2, cb)
eq_(self.result, "123")

# Now make it an iterator
result = ""
f = lambda x: func_with_callback(1, 2, x)
with nilmdb.utils.Iteratorizer(f) as it:
for i in it:
result += str(i)
eq_(result, "123")
eq_(it.retval, "return value")

# Make sure things work when an exception occurs
result = ""
with nilmdb.utils.Iteratorizer(
lambda x: func_with_callback(1, "a", x)) as it:
with assert_raises(TypeError) as e:
for i in it:
result += str(i)
eq_(result, "1a")

# Now try to trigger the case where we stop iterating
# mid-generator, and expect the iteratorizer to clean up after
# itself. This doesn't have a particular result in the test,
# but gains coverage.
def foo():
with nilmdb.utils.Iteratorizer(f) as it:
it.next()
foo()
eq_(it.retval, None)

# Do the same thing when the curl hack is applied
def foo():
with nilmdb.utils.Iteratorizer(f, curl_hack = True) as it:
it.next()
foo()
eq_(it.retval, None)

Loading…
Cancel
Save