Browse Source

Always use UTF-8 for filenames in nilmdb.bulkdata

tags/unicode
Jim Paris 11 years ago
parent
commit
41b3f3c018
1 changed files with 18 additions and 5 deletions
  1. +18
    -5
      nilmdb/bulkdata.py

+ 18
- 5
nilmdb/bulkdata.py View File

@@ -29,9 +29,18 @@ class BulkData(object):
def close(self):
self.getnode.cache_remove_all()

def create(self, path, layout_name):
def _encode_filename(self, path):
# Encode all paths to UTF-8, regardless of sys.getfilesystemencoding(),
# because we want to be able to represent all code points and the user
# will never be directly exposed to filenames. We can then do path
# manipulations on the UTF-8 directly.
if isinstance(path, unicode):
return path.encode('utf-8')
return path

def create(self, unicodepath, layout_name):
"""
path: path to the data (e.g. '/newton/prep').
unicodepath: path to the data (e.g. u'/newton/prep').
Paths must contain at least two elements, e.g.:
/newton/prep
/newton/raw
@@ -40,6 +49,8 @@ class BulkData(object):

layout_name: string for nilmdb.layout.get_named(), e.g. 'float32_8'
"""
path = self._encode_filename(unicodepath)

if path[0] != '/':
raise ValueError("paths must start with /")
[ group, node ] = path.rsplit("/", 1)
@@ -92,14 +103,15 @@ class BulkData(object):
raise ValueError("error creating table at that path: " + e.strerror)

# Open and cache it
self.getnode(path)
self.getnode(unicodepath)

# Success
return

def destroy(self, path):
def destroy(self, unicodepath):
"""Fully remove all data at a particular path. No way to undo
it! The group/path structure is removed, too."""
path = self._encode_filename(unicodepath)

# Get OS path
elements = path.lstrip('/').split('/')
@@ -125,9 +137,10 @@ class BulkData(object):
# Cache open tables
@nilmdb.utils.lru_cache(size = table_cache_size,
onremove = lambda x: x.close())
def getnode(self, path):
def getnode(self, unicodepath):
"""Return a Table object corresponding to the given database
path, which must exist."""
path = self._encode_filename(unicodepath)
elements = path.lstrip('/').split('/')
ospath = os.path.join(self.root, *elements)
return Table(ospath)


Loading…
Cancel
Save