Clean up and simplify Table.get_*, including __getitem__

nilmdb/server/bulkdata.py

@@ -392,6 +392,7 @@ class Table(object):
     def append(self, data):
         """Append the data and flush it to disk.
         data is a nested Python list [[row],[row],[...]]"""
+        raise Exception("don't call me")
         remaining = len(data)
         dataiter = iter(data)
         while remaining:
@@ -476,10 +477,9 @@ class Table(object):
             # Success, so update self.nrows accordingly
             self.nrows = tot_rows
 
-    def _get_data(self, start, stop, as_string):
+    def get_data(self, start, stop):
         """Extract data corresponding to Python range [n:m],
-        and returns a numeric list or formatted string,
-        depending on as_string."""
+        and returns a formatted string"""
         if (start is None or
             stop is None or
             start > stop or
@@ -495,42 +495,27 @@ class Table(object):
             if count > remaining:
                 count = remaining
             f = self.file_open(subdir, filename)
-            if as_string:
-                ret.append(f.extract_string(offset, count))
-            else:
-                ret.extend(f.extract_list(offset, count))
+            ret.append(f.extract_string(offset, count))
             remaining -= count
             row += count
-        if as_string:
-            return "".join(ret)
-        return ret
+        return "".join(ret)
 
-    def get_as_text(self, start, stop):
-        """Extract data corresponding to Python range [n:m],
-        and returns a formatted string"""
-        return self._get_data(start, stop, True)
+    def get_timestamp(self, row):
+        """Return the timestamp corresponding to the given row"""
+        if (row is None or row < 0 or row >= self.nrows):
+            raise IndexError("Index out of range")
+        return int(self[row].split(' ')[0])
 
     def __getitem__(self, key):
         """Extract data and return it.  Supports simple indexing
-        (table[n]) and range slices (table[n:m]).  Returns a nested
-        Python list [[row],[row],[...]]"""
+        (table[n]) and range slices (table[n:m]).  For simple
+        indexing, return 'row\n'.  For ranges, return a list of rows
+        ['row\n','row\n'].
 
-        # Handle simple slices
+        This is inefficient; use .get_data instead."""
         if isinstance(key, slice):
-            # Fall back to brute force if the slice isn't simple
-            try:
-                if (key.step is not None and key.step != 1):
-                    raise IndexError
-                return self._get_data(key.start, key.stop, False)
-            except IndexError:
-                return [ self[x] for x in xrange(*key.indices(self.nrows)) ]
-
-        # Handle single points (inefficiently!)
-        if key < 0 or key >= self.nrows:
-            raise IndexError("Index out of range")
-        (subdir, filename, offset, count) = self._offset_from_row(key)
-        f = self.file_open(subdir, filename)
-        return f.extract_list(offset, 1)[0]
+            return [ self[x] for x in xrange(*key.indices(self.nrows)) ]
+        return self.get_data(key, key+1)
 
     def _remove_rows(self, subdir, filename, start, stop):
         """Helper to mark specific rows as being removed from a
@@ -628,4 +613,4 @@ class TimestampOnlyTable(object):
     def __init__(self, table):
         self.table = table
     def __getitem__(self, index):
-        return self.table[index][0]
+        return self.table.get_timestamp(index)

nilmdb/server/nilmdb.py

@@ -539,10 +539,10 @@ class NilmDB(object):
             row_max = row_start + remaining
             if row_max < row_end:
                 row_end = row_max
-                restart = table[row_max][0]
+                restart = table.get_timestamp(row_max)
 
             # Gather these results up
-            result.append(table.get_as_text(row_start, row_end))
+            result.append(table.get_data(row_start, row_end))
 
             # Count them
             remaining -= row_end - row_start

tests/test_bulkdata.py

@@ -55,15 +55,16 @@ class TestBulkData(object):
             x = node[0]
         raw = []
         for i in range(1000):
-            raw.append([10000+i, 1, 2, 3, 4, 5, 6, 7, 8 ])
-        node.append(raw[0:1])
-        node.append(raw[1:100])
-        node.append(raw[100:])
+            raw.append("%d 1 2 3 4 5 6 7 8\n" % (10000 + i))
+        node.append_string("".join(raw[0:1]), 0, 50000)
+        node.append_string("".join(raw[1:100]), 0, 50000)
+        node.append_string("".join(raw[100:]), 0, 50000)
 
         misc_slices = [ 0, 100, slice(None), slice(0), slice(10),
                         slice(5,10), slice(3,None), slice(3,-3),
                         slice(20,10), slice(200,100,-1), slice(None,0,-1),
                         slice(100,500,5) ]
+
         # Extract slices
         for s in misc_slices:
             eq_(node[s], raw[s])
@@ -71,8 +72,8 @@ class TestBulkData(object):
         # Extract misc slices while appending, to make sure the
         # data isn't being added in the middle of the file
         for s in [2, slice(1,5), 2, slice(1,5)]:
-            node.append([[0,0,0,0,0,0,0,0,0]])
-            raw.append([0,0,0,0,0,0,0,0,0])
+            node.append_string("0 0 0 0 0 0 0 0 0\n", 0, 50000)
+            raw.append("0 0 0 0 0 0 0 0 0\n")
             eq_(node[s], raw[s])
 
         # Get some coverage of remove; remove is more fully tested