Browse Source

bulkdata: Use file writes instead of writing to the mmap.

Extending and then writing to the mmap file has a problem: if the disk
fills up, the mapping becomes invalid, and the Python interpreter will
get a SIGBUS, killing it.  It's difficult to catch this gracefully;
there's no way to do that with existing modules.  Instead, switch to
only using mmap when reading, and normal file writes when writing.
Since we only ever append, it should have similar performance.
Jim Paris 9 years ago
2 changed files with 69 additions and 40 deletions
  1. +68
  2. +1

+ 68
- 39
nilmdb/server/ View File

@@ -13,6 +13,16 @@ import struct
import mmap
import re

# If we have the faulthandler module, use it. All of the mmap stuff
# might trigger a SIGSEGV or SIGBUS if we're not careful, and
# faulthandler will give a traceback in that case. (the Python
# interpreter will still die either way).
try: # pragma: no cover
import faulthandler
except: # pragma: no cover

# Up to 256 open file descriptors at any given time.
# These variables are global so they can be used in the decorator arguments.
table_cache_size = 16
@@ -161,6 +171,52 @@ class BulkData(object):
ospath = os.path.join(self.root, *elements)
return Table(ospath)

@nilmdb.utils.must_close(wrap_verify = True)
class File(object):
"""Object representing a single file on disk. Data can be appended,
or the self.mmap handle can be used for random reads."""

def __init__(self, root, subdir, filename):
# Create path if it doesn't exist
os.mkdir(os.path.join(root, subdir))
except OSError:

# Open/create file
self._f = open(os.path.join(root, subdir, filename), "a+b", 0)

# Seek to end, and get size, 2)
self.size = self._f.tell()

# Open mmap object
self.mmap = None

def _mmap_reopen(self):
if self.size == 0:
# Don't mmap if the file is empty; it would fail
elif self.mmap is None:
# Not opened yet, so open it
self.mmap = mmap.mmap(self._f.fileno(), 0)
# Already opened, so just resize it

def close(self):
if self.mmap is not None:

def append(self, data):
# Write data, flush it, and resize our mmap accordingly
self.size += len(data)

@nilmdb.utils.must_close(wrap_verify = True)
class Table(object):
"""Tools to help access a single table (data at a specific OS path)."""
@@ -211,7 +267,7 @@ class Table(object):
self.nrows = self._get_nrows()

def close(self):

# Internal helpers
def _get_nrows(self):
@@ -275,37 +331,11 @@ class Table(object):

# Cache open files
@nilmdb.utils.lru_cache(size = fd_cache_size,
keys = slice(0, 3), # exclude newsize
onremove = lambda x: x.close())
def mmap_open(self, subdir, filename, newsize = None):
onremove = lambda f: f.close())
def file_open(self, subdir, filename):
"""Open and map a given 'subdir/filename' (relative to self.root).
Will be automatically closed when evicted from the cache.

If 'newsize' is provided, the file is truncated to the given
size before the mapping is returned. (Note that the LRU cache
on this function means the truncate will only happen if the
object isn't already cached; mmap.resize should be used too.)"""
os.mkdir(os.path.join(self.root, subdir))
except OSError:
f = open(os.path.join(self.root, subdir, filename), "a+", 0)
if newsize is not None:
# mmap can't map a zero-length file, so this allows the
# caller to set the filesize between file creation and
# mmap.
mm = mmap.mmap(f.fileno(), 0)
return mm

def mmap_open_resize(self, subdir, filename, newsize):
"""Open and map a given 'subdir/filename' (relative to self.root).
The file is resized to the given size."""
# Pass new size to mmap_open
mm = self.mmap_open(subdir, filename, newsize)
# In case we got a cached copy, need to call mm.resize too.
return mm
Will be automatically closed when evicted from the cache."""
return File(self.root, subdir, filename)

def append(self, data):
"""Append the data and flush it to disk.
@@ -317,14 +347,13 @@ class Table(object):
(subdir, fname, offset, count) = self._offset_from_row(self.nrows)
if count > remaining:
count = remaining
newsize = offset + count * self.packer.size
mm = self.mmap_open_resize(subdir, fname, newsize)

f = self.file_open(subdir, fname)

# Write the data
for i in xrange(count):
row =
remaining -= count
self.nrows += count

@@ -351,7 +380,7 @@ class Table(object):
(subdir, filename, offset, count) = self._offset_from_row(row)
if count > remaining:
count = remaining
mm = self.mmap_open(subdir, filename)
mm = self.file_open(subdir, filename).mmap
for i in xrange(count):
ret.append(list(self.packer.unpack_from(mm, offset)))
offset += self.packer.size
@@ -363,7 +392,7 @@ class Table(object):
if key < 0 or key >= self.nrows:
raise IndexError("Index out of range")
(subdir, filename, offset, count) = self._offset_from_row(key)
mm = self.mmap_open(subdir, filename)
mm = self.file_open(subdir, filename).mmap
# unpack_from ignores the mmap object's current seek position
return list(self.packer.unpack_from(mm, offset))

@@ -410,8 +439,8 @@ class Table(object):
# are generally easier if we don't have to special-case that.
if (len(merged) == 1 and
merged[0][0] == 0 and merged[0][1] == self.rows_per_file):
# Close potentially open file in mmap_open LRU cache
self.mmap_open.cache_remove(self, subdir, filename)
# Close potentially open file in file_open LRU cache
self.file_open.cache_remove(self, subdir, filename)

# Delete files

+ 1
- 1
nilmdb/server/ View File

@@ -396,7 +396,7 @@ class NilmDB(object):
path: Path at which to add the data
start: Starting timestamp
end: Ending timestamp
data: Rows of data, to be passed to PyTable's table.append
data: Rows of data, to be passed to bulkdata table.append
method. E.g.
# First check for basic overlap using timestamp info given.