Browse Source

More work on BulkData

tags/replace-pytables
Jim Paris 9 years ago
parent
commit
3816645313
1 changed files with 47 additions and 23 deletions
  1. +47
    -23
      nilmdb/bulkdata.py

+ 47
- 23
nilmdb/bulkdata.py View File

@@ -4,11 +4,16 @@ from __future__ import absolute_import
import nilmdb
from nilmdb.printf import *

import tables
from nilmdb.lrucache import lrucache

import os
import sys
import cPickle as pickle

# Up to 256 open file descriptors at any given time
table_cache_size = 16
fd_cache_size = 16

class BulkData(object):
def __init__(self, basepath):
self.basepath = basepath
@@ -19,7 +24,6 @@ class BulkData(object):
os.mkdir(self.root)

self.opened = True
self.tablecache = {}

def __del__(self):
if "opened" in self.__dict__: # pragma: no cover
@@ -28,8 +32,7 @@ class BulkData(object):
self.basepath)

def close(self):
for table in self.tablecache.values():
table.close()
self.getnode.cache_remove_all()
del self.opened

def create(self, path, layout_name):
@@ -78,10 +81,10 @@ class BulkData(object):
elements = path.lstrip('/').split('/')
for i in range(len(elements)):
ospath = os.path.join(self.root, *elements[0:i])
if not os.path.isdir(ospath):
os.mkdir(ospath)
if os.path.isfile(os.path.join(ospath, "format")):
raise ValueError("path is subdir of existing node")
if not os.path.isdir(ospath):
os.mkdir(ospath)

# Make the final dir
ospath = os.path.join(self.root, *elements)
@@ -96,7 +99,7 @@ class BulkData(object):
raise ValueError("error creating table at that path: " + e.strerror)

# Open and cache it
self.getnode(path)
self.getnode(ospath)

# Success
return
@@ -104,24 +107,34 @@ class BulkData(object):
def destroy(self, path):
"""Fully remove all data at a particular path. No way to undo
it! The group/path structure is removed, too."""
# Delete the data node, and all parent nodes (if they have no
# remaining children)

### XXX TODO: Remove path recursively, then try to rmdir on all parents
### up to self.root or until we hit an error, whichever is first.
split_path = path.lstrip('/').split("/")
while split_path:
name = split_path.pop()
where = "/" + "/".join(split_path)

# Get OS path
elements = path.lstrip('/').split('/')
ospath = os.path.join(self.root, *elements)

# Remove Table object from cache
self.getnode.cache_remove(self, ospath)

# Remove the contents of the target directory
if not os.path.isfile(os.path.join(ospath, "format")):
raise ValueError("nothing at that path")
for file in os.listdir(ospath):
os.remove(os.path.join(ospath, file))

# Remove empty parent directories
for i in reversed(range(len(elements))):
ospath = os.path.join(self.root, *elements[0:i])
try:
self.h5file.removeNode(where, name, recursive = False)
except tables.NodeError:
break
os.rmdir(ospath)
except OSError:
pass

# Cache open tables
@lrucache(size = table_cache_size, onremove = lambda x: x.close())
def getnode(self, path):
if path not in self.tablecache:
self.tablecache[path] = Table(path)
return self.tablecache[path]
"""Return a Table object corresponding to the given path,
which must exist."""
return Table(path)

class Table(object):
"""Tools to help access a single table (data at a specific path)"""
@@ -129,18 +142,28 @@ class Table(object):
def __init__(self, path):
self.path = path

def close(self):
# Cache open files
@lrucache(size = fd_cache_size, onremove = lambda x: self.mmap_close(x))
def mmap_open(self, file):
"""??"""
pass

def mmap_close(self, file):

def close(self):
self.mmap_open.cache_remove_all()

@property
def nrows(self):
"""Return number of rows total"""
return 0
raise NotImplementedError()
return int(self.table.nrows)

def append(self, data):
"""Append the data and flush it to disk.
data is a nested Python list [[row],[row],[...]]"""
return 0
raise NotImplementedError()
self.table.append(data)
self.table.flush()
@@ -149,6 +172,7 @@ class Table(object):
"""Needs to support simple indexing (table[n]) and
range slices (table[n:m]). Returns a nested Python
list [[row],[row],[...]]"""
return []
raise NotImplementedError()
return self.table.__getitem__(val)



Loading…
Cancel
Save