Browse Source

Bulkdata extract works now.

tags/replace-pytables
Jim Paris 9 years ago
parent
commit
1beae5024e
1 changed files with 41 additions and 11 deletions
  1. +41
    -11
      nilmdb/bulkdata.py

+ 41
- 11
nilmdb/bulkdata.py View File

@@ -116,11 +116,11 @@ class BulkData(object):

# Remove empty parent directories
for i in reversed(range(len(elements))):
ospath = os.path.join(self.root, *elements[0:i])
ospath = os.path.join(self.root, *elements[0:i+1])
try:
os.rmdir(ospath)
except OSError:
pass
break

# Cache open tables
@nilmdb.utils.lru_cache(size = table_cache_size,
@@ -243,20 +243,50 @@ class Table(object):
mm.resize(newsize)

# Write the data
for i in range(count):
for i in xrange(count):
row = dataiter.next()
mm.write(self.packer.pack(*row))
remaining -= count
self.nrows += count

def __getitem__(self, val):
"""Needs to support simple indexing (table[n]) and
range slices (table[n:m]). Returns a nested Python
list [[row],[row],[...]]"""
### TODO: actually read the data.
return []
raise NotImplementedError()
return self.table.__getitem__(val)
def __getitem__(self, key):
"""Extract data and return it. Supports simple indexing
(table[n]) and range slices (table[n:m]). Returns a nested
Python list [[row],[row],[...]]"""

# Handle simple slices
if isinstance(key, slice):
# Fall back to brute force if the slice isn't simple
if ((key.step is not None and key.step != 1) or
key.start is None or
key.stop is None or
key.start >= key.stop or
key.start < 0 or
key.stop > self.nrows):
return [ self[x] for x in xrange(*key.indices(self.nrows)) ]

ret = []
row = key.start
remaining = key.stop - key.start
while remaining:
(filename, offset, count) = self._fnoffset_from_row(row)
if count > remaining:
count = remaining
mm = self.mmap_open(filename)
for i in xrange(count):
ret.append(list(self.packer.unpack_from(mm, offset)))
offset += self.packer.size
remaining -= count
row += count
return ret

# Handle single points
if key < 0 or key >= self.nrows:
raise IndexError("Index out of range")
(filename, offset, count) = self._fnoffset_from_row(key)
mm = self.mmap_open(filename)
# unpack_from ignores the mmap object's current seek position
return self.packer.unpack_from(mm, offset)

class TimestampOnlyTable(object):
"""Helper that lets us pass a Tables object into bisect, by


Loading…
Cancel
Save