Browse Source

Clean up and simplify Table.get_*, including __getitem__

tags/nilmdb-1.4.0
Jim Paris 8 years ago
parent
commit
fb4f4519ff
3 changed files with 26 additions and 40 deletions
  1. +17
    -32
      nilmdb/server/bulkdata.py
  2. +2
    -2
      nilmdb/server/nilmdb.py
  3. +7
    -6
      tests/test_bulkdata.py

+ 17
- 32
nilmdb/server/bulkdata.py View File

@@ -392,6 +392,7 @@ class Table(object):
def append(self, data):
"""Append the data and flush it to disk.
data is a nested Python list [[row],[row],[...]]"""
raise Exception("don't call me")
remaining = len(data)
dataiter = iter(data)
while remaining:
@@ -476,10 +477,9 @@ class Table(object):
# Success, so update self.nrows accordingly
self.nrows = tot_rows

def _get_data(self, start, stop, as_string):
def get_data(self, start, stop):
"""Extract data corresponding to Python range [n:m],
and returns a numeric list or formatted string,
depending on as_string."""
and returns a formatted string"""
if (start is None or
stop is None or
start > stop or
@@ -495,42 +495,27 @@ class Table(object):
if count > remaining:
count = remaining
f = self.file_open(subdir, filename)
if as_string:
ret.append(f.extract_string(offset, count))
else:
ret.extend(f.extract_list(offset, count))
ret.append(f.extract_string(offset, count))
remaining -= count
row += count
if as_string:
return "".join(ret)
return ret
return "".join(ret)

def get_as_text(self, start, stop):
"""Extract data corresponding to Python range [n:m],
and returns a formatted string"""
return self._get_data(start, stop, True)
def get_timestamp(self, row):
"""Return the timestamp corresponding to the given row"""
if (row is None or row < 0 or row >= self.nrows):
raise IndexError("Index out of range")
return int(self[row].split(' ')[0])

def __getitem__(self, key):
"""Extract data and return it. Supports simple indexing
(table[n]) and range slices (table[n:m]). Returns a nested
Python list [[row],[row],[...]]"""
(table[n]) and range slices (table[n:m]). For simple
indexing, return 'row\n'. For ranges, return a list of rows
['row\n','row\n'].

# Handle simple slices
This is inefficient; use .get_data instead."""
if isinstance(key, slice):
# Fall back to brute force if the slice isn't simple
try:
if (key.step is not None and key.step != 1):
raise IndexError
return self._get_data(key.start, key.stop, False)
except IndexError:
return [ self[x] for x in xrange(*key.indices(self.nrows)) ]

# Handle single points (inefficiently!)
if key < 0 or key >= self.nrows:
raise IndexError("Index out of range")
(subdir, filename, offset, count) = self._offset_from_row(key)
f = self.file_open(subdir, filename)
return f.extract_list(offset, 1)[0]
return [ self[x] for x in xrange(*key.indices(self.nrows)) ]
return self.get_data(key, key+1)

def _remove_rows(self, subdir, filename, start, stop):
"""Helper to mark specific rows as being removed from a
@@ -628,4 +613,4 @@ class TimestampOnlyTable(object):
def __init__(self, table):
self.table = table
def __getitem__(self, index):
return self.table[index][0]
return self.table.get_timestamp(index)

+ 2
- 2
nilmdb/server/nilmdb.py View File

@@ -539,10 +539,10 @@ class NilmDB(object):
row_max = row_start + remaining
if row_max < row_end:
row_end = row_max
restart = table[row_max][0]
restart = table.get_timestamp(row_max)

# Gather these results up
result.append(table.get_as_text(row_start, row_end))
result.append(table.get_data(row_start, row_end))

# Count them
remaining -= row_end - row_start


+ 7
- 6
tests/test_bulkdata.py View File

@@ -55,15 +55,16 @@ class TestBulkData(object):
x = node[0]
raw = []
for i in range(1000):
raw.append([10000+i, 1, 2, 3, 4, 5, 6, 7, 8 ])
node.append(raw[0:1])
node.append(raw[1:100])
node.append(raw[100:])
raw.append("%d 1 2 3 4 5 6 7 8\n" % (10000 + i))
node.append_string("".join(raw[0:1]), 0, 50000)
node.append_string("".join(raw[1:100]), 0, 50000)
node.append_string("".join(raw[100:]), 0, 50000)

misc_slices = [ 0, 100, slice(None), slice(0), slice(10),
slice(5,10), slice(3,None), slice(3,-3),
slice(20,10), slice(200,100,-1), slice(None,0,-1),
slice(100,500,5) ]

# Extract slices
for s in misc_slices:
eq_(node[s], raw[s])
@@ -71,8 +72,8 @@ class TestBulkData(object):
# Extract misc slices while appending, to make sure the
# data isn't being added in the middle of the file
for s in [2, slice(1,5), 2, slice(1,5)]:
node.append([[0,0,0,0,0,0,0,0,0]])
raw.append([0,0,0,0,0,0,0,0,0])
node.append_string("0 0 0 0 0 0 0 0 0\n", 0, 50000)
raw.append("0 0 0 0 0 0 0 0 0\n")
eq_(node[s], raw[s])

# Get some coverage of remove; remove is more fully tested


Loading…
Cancel
Save