Use the new NumpyClient for extracting data in filter

setup.py

@@ -61,7 +61,7 @@ setup(name='nilmtools',
       long_description = "NILM Database Tools",
       license = "Proprietary",
       author_email = 'jim@jtan.com',
-      install_requires = [ 'nilmdb >= 1.4.6',
+      install_requires = [ 'nilmdb >= 1.4.8',
                            'numpy',
                            'scipy',
                            'matplotlib',

src/decimate.py

@@ -71,7 +71,7 @@ def decimate(data, interval, args, insert_function, final):
     data = data[:n,:]
 
     # Reshape it into 3D so we can process 'factor' rows at a time
-    data.shape = (n // factor, factor, m)
+    data = data.reshape(n // factor, factor, m)
 
     # Fill the result
     out = np.c_[ np.mean(data[:,:,mean_col], axis=1),

src/filter.py

@@ -4,6 +4,7 @@ from __future__ import absolute_import
 
 import nilmdb.client
 from nilmdb.client import Client
+from nilmdb.client.numpyclient import NumpyClient
 from nilmdb.utils.printf import *
 from nilmdb.utils.time import (parse_time, timestamp_to_human,
                                timestamp_to_seconds)
@@ -277,8 +278,8 @@ class Filter(object):
         """
         if args is None:
             args = []
-        extractor = Client(self.src.url).stream_extract
-        inserter = Client(self.dest.url).stream_insert_context
+        extractor = NumpyClient(self.src.url).stream_extract_numpy
+        inserter = NumpyClient(self.dest.url).stream_insert_context
 
         # Format output data.
         formatter = lambda row: " ".join([repr(x) for x in row]) + "\n"
@@ -299,15 +300,11 @@ class Filter(object):
                     np.savetxt(s, array)
                     insert_ctx.insert(s.getvalue())
 
-                extract = extractor(self.src.path, interval.start, interval.end)
                 old_array = np.array([])
-                for batched in batch(extract, rows):
-                    # Read in this batch of data.  This turns out to
-                    # be a very fast way to read and convert it (order
-                    # of magnitude faster than numpy.loadtxt)
-                    new_array = np.fromstring("\n".join(batched), sep=' ')
-                    new_array = new_array.reshape(-1, self.src.total_count)
-
+                for new_array in extractor(self.src.path,
+                                           interval.start, interval.end,
+                                           layout = self.src.layout,
+                                           maxrows = rows):
                     # If we still had old data left, combine it
                     if old_array.shape[0] != 0:
                         array = np.vstack((old_array, new_array))