WIP switching away from PyTables

nilmdb/bulkdata.py

@@ -7,13 +7,17 @@ from nilmdb.printf import *
 import tables
 import os
 import sys
+import re
 
 class BulkData(object):
     def __init__(self, basepath):
         self.basepath = basepath
+        self.root = self.basepath + "/data"
+
+        # Make root path
+        if not os.path.isdir(self.root):
+            os.mkdir(self.root)
 
-        h5filename = os.path.abspath(self.basepath + "/data.h5")
-        self.h5file = tables.openFile(h5filename, "a", "NILM Database")
         self.opened = True
         self.tablecache = {}
 
@@ -24,12 +28,10 @@ class BulkData(object):
                     self.basepath)
 
     def close(self):
-        self.h5file.close()
         for table in self.tablecache.values():
             table.close()
         del self.opened
 
-
     def create(self, path, layout_name):
         """
         path: path to the data (e.g. '/newton/prep').
@@ -47,43 +49,57 @@ class BulkData(object):
         if group == '':
             raise ValueError("invalid path")
 
-        # Get description
+        # Get layout, and build format string for struct module
         try:
-            # Build PyTables description
             layout = nilmdb.layout.get_named(layout_name)
-            desc = {}
-            desc['timestamp'] = tables.Col.from_type('float64', pos=0)
+            desc = '<d'  # Little endian, double timestamp
+            mapping = {
+                "int8": 'b',
+                "uint8": 'B',
+                "int16": 'h',
+                "uint16": 'H',
+                "int32": 'i',
+                "uint32": 'I',
+                "int64": 'q',
+                "uint64": 'Q',
+                "float32": 'f',
+                "float64": 'd',
+                }
             for n in range(layout.count):
-                desc['c' + str(n+1)] = tables.Col.from_type(layout.datatype,
-                                                            pos=n+1)
-            desc = tables.Description(desc)
+                desc += mapping[layout.datatype]
         except KeyError:
-            raise ValueError("no such layout")
+            raise ValueError("no such layout, or bad data types")
 
-        # Estimated table size (for PyTables optimization purposes): assume
-        # 3 months worth of data at 8 KHz.  It's OK if this is wrong.
-        exp_rows = 8000 * 60*60*24*30*3
+        ## XXXX TODO: Fix inside this "try:" so for
+        # path /foo/bar, we can't create /foo/bar/baz:
 
         # Create the table
         try:
-            table = self.h5file.createTable(group,
-                                            node,
-                                            description = desc,
-                                            expectedrows = exp_rows,
-                                            createparents = True)
-        except AttributeError:
-            # Trying to create e.g. /foo/bar/baz when /foo/bar is already
-            # a table raises this error.
+            # Create path
+            if os.path.isdir(self.root + path):
+                raise OSError()
+            os.makedirs(self.root + path)
+
+            # Write format string to file
+            with open(self.root + path + "/format", "w") as f:
+                f.write(desc + "\n")
+        except OSError:
             raise ValueError("error creating table at that path")
 
+        # Open and cache it
+        self.getnode(path)
+
         # Success
         return
 
     def destroy(self, path):
         """Fully remove all data at a particular path.  No way to undo
-        it!  The group structure is removed, too."""
+        it!  The group/path structure is removed, too."""
         # Delete the data node, and all parent nodes (if they have no
         # remaining children)
+
+        ### XXX TODO: Remove path recursively, then try to rmdir on all parents
+        ### up to self.root or until we hit an error, whichever is first.
         split_path = path.lstrip('/').split("/")
         while split_path:
             name = split_path.pop()
@@ -131,8 +147,3 @@ class TimestampOnlyTable(object):
         self.table = table
     def __getitem__(self, index):
         return self.table[index][0]
-
-# Just a helper to so we can call bulkdata.open() instead of
-# bulkdata.BulkData():
-def open(path):
-    return BulkData(path)

nilmdb/nilmdb.py

@@ -104,7 +104,7 @@ class NilmDB(object):
                 raise IOError("can't create tree " + self.basepath)
 
         # Our data goes inside it
-        self.data = bulkdata.open(self.basepath)
+        self.data = bulkdata.BulkData(self.basepath)
 
         # SQLite database too
         sqlfilename = os.path.abspath(self.basepath + "/data.sql")