Browse Source

Fully replace bulkdata.File with rocket.Rocket

tags/nilmdb-1.3.0
Jim Paris 11 years ago
parent
commit
b0f12d55dd
1 changed files with 12 additions and 37 deletions
  1. +12
    -37
      nilmdb/server/bulkdata.py

+ 12
- 37
nilmdb/server/bulkdata.py View File

@@ -152,38 +152,6 @@ class BulkData(object):
ospath = os.path.join(self.root, *elements)
return Table(ospath)

@nilmdb.utils.must_close(wrap_verify = False)
class File(object):
"""Object representing a single file on disk. Data can be appended
or extracted using the rocket functions."""

def __init__(self, root, subdir, filename, layout):
# Create path if it doesn't exist
try:
os.mkdir(os.path.join(root, subdir))
except OSError:
pass

# Open/create file (through rocket interface)
fullpath = os.path.join(root, subdir, filename)
self.rocket = rocket.Rocket(layout, fullpath)

def close(self):
self.rocket.close()

@property
def size(self):
return self.rocket.file_size

def append_rocket_iter(self, maxrows, dataiter):
return self.rocket.append_iter(maxrows, dataiter)

def extract_rocket_list(self, offset, rows):
return self.rocket.extract_list(offset, rows)

def extract_rocket_string(self, offset, rows):
return self.rocket.extract_string(offset, rows)

@nilmdb.utils.must_close(wrap_verify = False)
class Table(object):
"""Tools to help access a single table (data at a specific OS path)."""
@@ -319,7 +287,14 @@ class Table(object):
def file_open(self, subdir, filename):
"""Open and map a given 'subdir/filename' (relative to self.root).
Will be automatically closed when evicted from the cache."""
return File(self.root, subdir, filename, self.layout)
# Create path if it doesn't exist
try:
os.mkdir(os.path.join(self.root, subdir))
except OSError:
pass
# Return a rocket.Rocket object, which contains the open file
return rocket.Rocket(self.layout,
os.path.join(self.root, subdir, filename))

def append(self, data):
"""Append the data and flush it to disk.
@@ -335,7 +310,7 @@ class Table(object):
f = self.file_open(subdir, fname)

# Write the data
written = f.append_rocket_iter(count, dataiter)
written = f.append_iter(count, dataiter)
if written != count: # pragma: no cover
raise Exception("Didn't write the expected number of rows: "
+ str(written) + " != " + str(count))
@@ -362,9 +337,9 @@ class Table(object):
count = remaining
f = self.file_open(subdir, filename)
if as_string:
ret.append(f.extract_rocket_string(offset, count))
ret.append(f.extract_string(offset, count))
else:
ret.extend(f.extract_rocket_list(offset, count))
ret.extend(f.extract_list(offset, count))
remaining -= count
row += count
if as_string:
@@ -396,7 +371,7 @@ class Table(object):
raise IndexError("Index out of range")
(subdir, filename, offset, count) = self._offset_from_row(key)
f = self.file_open(subdir, filename)
return f.extract_rocket_list(offset, 1)[0]
return f.extract_list(offset, 1)[0]

def _remove_rows(self, subdir, filename, start, stop):
"""Helper to mark specific rows as being removed from a


Loading…
Cancel
Save