|
|
@@ -43,6 +43,12 @@ class BulkData(object): |
|
|
|
# 32768 files per dir should work even on FAT32 |
|
|
|
self.files_per_dir = 32768 |
|
|
|
|
|
|
|
if "initial_nrows" in kwargs: |
|
|
|
self.initial_nrows = kwargs["initial_nrows"] |
|
|
|
else: |
|
|
|
# First row is 0 |
|
|
|
self.initial_nrows = 0 |
|
|
|
|
|
|
|
# Make root path |
|
|
|
if not os.path.isdir(self.root): |
|
|
|
os.mkdir(self.root) |
|
|
@@ -254,7 +260,7 @@ class BulkData(object): |
|
|
|
path = self._encode_filename(unicodepath) |
|
|
|
elements = path.lstrip('/').split('/') |
|
|
|
ospath = os.path.join(self.root, *elements) |
|
|
|
return Table(ospath) |
|
|
|
return Table(ospath, self.initial_nrows) |
|
|
|
|
|
|
|
@nilmdb.utils.must_close(wrap_verify = False) |
|
|
|
class Table(object): |
|
|
@@ -291,9 +297,10 @@ class Table(object): |
|
|
|
pickle.dump(fmt, f, 2) |
|
|
|
|
|
|
|
# Normal methods |
|
|
|
def __init__(self, root): |
|
|
|
def __init__(self, root, initial_nrows): |
|
|
|
"""'root' is the full OS path to the directory of this table""" |
|
|
|
self.root = root |
|
|
|
self.initial_nrows = initial_nrows |
|
|
|
|
|
|
|
# Load the format |
|
|
|
with open(os.path.join(self.root, "_format"), "rb") as f: |
|
|
@@ -353,8 +360,14 @@ class Table(object): |
|
|
|
# Convert to row number |
|
|
|
return self._row_from_offset(subdir, filename, offset) |
|
|
|
|
|
|
|
# No files, so no data |
|
|
|
return 0 |
|
|
|
# No files, so no data. We typically start at row 0 in this |
|
|
|
# case, although initial_nrows is specified during some tests |
|
|
|
# to exercise other parts of the code better. Since we have |
|
|
|
# no files yet, round initial_nrows up so it points to a row |
|
|
|
# that would begin a new file. |
|
|
|
nrows = ((self.initial_nrows + (self.rows_per_file - 1)) // |
|
|
|
self.rows_per_file) * self.rows_per_file |
|
|
|
return nrows |
|
|
|
|
|
|
|
def _offset_from_row(self, row): |
|
|
|
"""Return a (subdir, filename, offset, count) tuple: |
|
|
|