|
|
@@ -152,38 +152,6 @@ class BulkData(object): |
|
|
|
ospath = os.path.join(self.root, *elements) |
|
|
|
return Table(ospath) |
|
|
|
|
|
|
|
@nilmdb.utils.must_close(wrap_verify = False) |
|
|
|
class File(object): |
|
|
|
"""Object representing a single file on disk. Data can be appended |
|
|
|
or extracted using the rocket functions.""" |
|
|
|
|
|
|
|
def __init__(self, root, subdir, filename, layout): |
|
|
|
# Create path if it doesn't exist |
|
|
|
try: |
|
|
|
os.mkdir(os.path.join(root, subdir)) |
|
|
|
except OSError: |
|
|
|
pass |
|
|
|
|
|
|
|
# Open/create file (through rocket interface) |
|
|
|
fullpath = os.path.join(root, subdir, filename) |
|
|
|
self.rocket = rocket.Rocket(layout, fullpath) |
|
|
|
|
|
|
|
def close(self): |
|
|
|
self.rocket.close() |
|
|
|
|
|
|
|
@property |
|
|
|
def size(self): |
|
|
|
return self.rocket.file_size |
|
|
|
|
|
|
|
def append_rocket_iter(self, maxrows, dataiter): |
|
|
|
return self.rocket.append_iter(maxrows, dataiter) |
|
|
|
|
|
|
|
def extract_rocket_list(self, offset, rows): |
|
|
|
return self.rocket.extract_list(offset, rows) |
|
|
|
|
|
|
|
def extract_rocket_string(self, offset, rows): |
|
|
|
return self.rocket.extract_string(offset, rows) |
|
|
|
|
|
|
|
@nilmdb.utils.must_close(wrap_verify = False) |
|
|
|
class Table(object): |
|
|
|
"""Tools to help access a single table (data at a specific OS path).""" |
|
|
@@ -319,7 +287,14 @@ class Table(object): |
|
|
|
def file_open(self, subdir, filename): |
|
|
|
"""Open and map a given 'subdir/filename' (relative to self.root). |
|
|
|
Will be automatically closed when evicted from the cache.""" |
|
|
|
return File(self.root, subdir, filename, self.layout) |
|
|
|
# Create path if it doesn't exist |
|
|
|
try: |
|
|
|
os.mkdir(os.path.join(self.root, subdir)) |
|
|
|
except OSError: |
|
|
|
pass |
|
|
|
# Return a rocket.Rocket object, which contains the open file |
|
|
|
return rocket.Rocket(self.layout, |
|
|
|
os.path.join(self.root, subdir, filename)) |
|
|
|
|
|
|
|
def append(self, data): |
|
|
|
"""Append the data and flush it to disk. |
|
|
@@ -335,7 +310,7 @@ class Table(object): |
|
|
|
f = self.file_open(subdir, fname) |
|
|
|
|
|
|
|
# Write the data |
|
|
|
written = f.append_rocket_iter(count, dataiter) |
|
|
|
written = f.append_iter(count, dataiter) |
|
|
|
if written != count: # pragma: no cover |
|
|
|
raise Exception("Didn't write the expected number of rows: " |
|
|
|
+ str(written) + " != " + str(count)) |
|
|
@@ -362,9 +337,9 @@ class Table(object): |
|
|
|
count = remaining |
|
|
|
f = self.file_open(subdir, filename) |
|
|
|
if as_string: |
|
|
|
ret.append(f.extract_rocket_string(offset, count)) |
|
|
|
ret.append(f.extract_string(offset, count)) |
|
|
|
else: |
|
|
|
ret.extend(f.extract_rocket_list(offset, count)) |
|
|
|
ret.extend(f.extract_list(offset, count)) |
|
|
|
remaining -= count |
|
|
|
row += count |
|
|
|
if as_string: |
|
|
@@ -396,7 +371,7 @@ class Table(object): |
|
|
|
raise IndexError("Index out of range") |
|
|
|
(subdir, filename, offset, count) = self._offset_from_row(key) |
|
|
|
f = self.file_open(subdir, filename) |
|
|
|
return f.extract_rocket_list(offset, 1)[0] |
|
|
|
return f.extract_list(offset, 1)[0] |
|
|
|
|
|
|
|
def _remove_rows(self, subdir, filename, start, stop): |
|
|
|
"""Helper to mark specific rows as being removed from a |
|
|
|