Browse Source

WIP switching away from PyTables

tags/replace-pytables
Jim Paris 9 years ago
parent
commit
c890ea93cb
2 changed files with 41 additions and 30 deletions
  1. +40
    -29
      nilmdb/bulkdata.py
  2. +1
    -1
      nilmdb/nilmdb.py

+ 40
- 29
nilmdb/bulkdata.py View File

@@ -7,13 +7,17 @@ from nilmdb.printf import *
import tables
import os
import sys
import re

class BulkData(object):
def __init__(self, basepath):
self.basepath = basepath
self.root = self.basepath + "/data"

# Make root path
if not os.path.isdir(self.root):
os.mkdir(self.root)

h5filename = os.path.abspath(self.basepath + "/data.h5")
self.h5file = tables.openFile(h5filename, "a", "NILM Database")
self.opened = True
self.tablecache = {}

@@ -24,12 +28,10 @@ class BulkData(object):
self.basepath)

def close(self):
self.h5file.close()
for table in self.tablecache.values():
table.close()
del self.opened


def create(self, path, layout_name):
"""
path: path to the data (e.g. '/newton/prep').
@@ -47,43 +49,57 @@ class BulkData(object):
if group == '':
raise ValueError("invalid path")

# Get description
# Get layout, and build format string for struct module
try:
# Build PyTables description
layout = nilmdb.layout.get_named(layout_name)
desc = {}
desc['timestamp'] = tables.Col.from_type('float64', pos=0)
desc = '<d' # Little endian, double timestamp
mapping = {
"int8": 'b',
"uint8": 'B',
"int16": 'h',
"uint16": 'H',
"int32": 'i',
"uint32": 'I',
"int64": 'q',
"uint64": 'Q',
"float32": 'f',
"float64": 'd',
}
for n in range(layout.count):
desc['c' + str(n+1)] = tables.Col.from_type(layout.datatype,
pos=n+1)
desc = tables.Description(desc)
desc += mapping[layout.datatype]
except KeyError:
raise ValueError("no such layout")
raise ValueError("no such layout, or bad data types")

# Estimated table size (for PyTables optimization purposes): assume
# 3 months worth of data at 8 KHz. It's OK if this is wrong.
exp_rows = 8000 * 60*60*24*30*3
## XXXX TODO: Fix inside this "try:" so for
# path /foo/bar, we can't create /foo/bar/baz:

# Create the table
try:
table = self.h5file.createTable(group,
node,
description = desc,
expectedrows = exp_rows,
createparents = True)
except AttributeError:
# Trying to create e.g. /foo/bar/baz when /foo/bar is already
# a table raises this error.
# Create path
if os.path.isdir(self.root + path):
raise OSError()
os.makedirs(self.root + path)

# Write format string to file
with open(self.root + path + "/format", "w") as f:
f.write(desc + "\n")
except OSError:
raise ValueError("error creating table at that path")

# Open and cache it
self.getnode(path)

# Success
return

def destroy(self, path):
"""Fully remove all data at a particular path. No way to undo
it! The group structure is removed, too."""
it! The group/path structure is removed, too."""
# Delete the data node, and all parent nodes (if they have no
# remaining children)

### XXX TODO: Remove path recursively, then try to rmdir on all parents
### up to self.root or until we hit an error, whichever is first.
split_path = path.lstrip('/').split("/")
while split_path:
name = split_path.pop()
@@ -131,8 +147,3 @@ class TimestampOnlyTable(object):
self.table = table
def __getitem__(self, index):
return self.table[index][0]

# Just a helper to so we can call bulkdata.open() instead of
# bulkdata.BulkData():
def open(path):
return BulkData(path)

+ 1
- 1
nilmdb/nilmdb.py View File

@@ -104,7 +104,7 @@ class NilmDB(object):
raise IOError("can't create tree " + self.basepath)

# Our data goes inside it
self.data = bulkdata.open(self.basepath)
self.data = bulkdata.BulkData(self.basepath)

# SQLite database too
sqlfilename = os.path.abspath(self.basepath + "/data.sql")


Loading…
Cancel
Save