|
|
@@ -316,7 +316,8 @@ class Filter(object): |
|
|
|
self._client_dest.stream_update_metadata(self.dest.path, data) |
|
|
|
|
|
|
|
# The main filter processing method. |
|
|
|
def process_numpy(self, function, args = None, rows = 100000): |
|
|
|
def process_numpy(self, function, args = None, rows = 100000, |
|
|
|
intervals = None): |
|
|
|
"""Calls process_numpy_interval for each interval that currently |
|
|
|
exists in self.src, but doesn't exist in self.dest. It will |
|
|
|
process the data in chunks as follows: |
|
|
@@ -325,6 +326,9 @@ class Filter(object): |
|
|
|
corresponding to the data. The data is converted to a Numpy |
|
|
|
array in chunks of 'rows' rows at a time. |
|
|
|
|
|
|
|
If 'intervals' is not None, process those intervals instead of |
|
|
|
the default list. |
|
|
|
|
|
|
|
'function' should be defined as: |
|
|
|
# def function(data, interval, args, insert_func, final) |
|
|
|
|
|
|
@@ -358,7 +362,7 @@ class Filter(object): |
|
|
|
maxrows = rows) |
|
|
|
inserter_func = functools.partial(inserter, self.dest.path) |
|
|
|
|
|
|
|
for interval in self.intervals(): |
|
|
|
for interval in (intervals or self.intervals()): |
|
|
|
print "Processing", interval.human_string() |
|
|
|
process_numpy_interval(interval, extractor_func, inserter_func, |
|
|
|
rows * 3, function, args) |
|
|
|