|
|
@@ -367,8 +367,11 @@ class Filter(object): |
|
|
|
extract = extractor(self.src.path, interval.start, interval.end) |
|
|
|
old_array = np.array([]) |
|
|
|
for batched in batch(extract, rows): |
|
|
|
# Read in this batch of data |
|
|
|
new_array = np.loadtxt(batched) |
|
|
|
# Read in this batch of data. This turns out to |
|
|
|
# be a very fast way to read and convert it (order |
|
|
|
# of magnitude faster than numpy.loadtxt) |
|
|
|
new_array = np.fromstring("".join(batched), sep=' ') |
|
|
|
new_array = new_array.reshape(-1, self.src.total_count) |
|
|
|
|
|
|
|
# If we still had old data left, combine it |
|
|
|
if old_array.shape[0] != 0: |
|
|
|