Use 'repr' instead of 'str' in Interval string representation.

Otherwise timestamps get truncated to 2 decimal places.

.gitignore

@@ -0,0 +1,4 @@
+db/
+tests/*testdb/
+.coverage
+*.pyc

README.txt

@@ -1,2 +1,4 @@
 sudo apt-get install python-nose python-coverage
-sudo apt-get install python-tables cython python-cherrypy3
+sudo apt-get install python-tables python-cherrypy3
+sudo apt-get install cython # 0.17.1-1 or newer
+

TODO

@@ -1,5 +1 @@
 - Merge adjacent intervals on insert (maybe with client help?)
-
-- Better testing:
-  - see about getting coverage on layout.pyx
-  - layout.pyx performance tests, before and after generalization

design.md

@@ -161,6 +161,18 @@ IntervalSet speed
 - Might be algorithmic improvements to be made in Interval.py,
   like in `__and__`
 
+- Replaced again with rbtree.  Seems decent.  Numbers are time per
+  insert for 2**17 insertions, followed by total wall time and RAM
+  usage for running "make test" with `test_rbtree` and `test_interval`
+  with range(5,20):
+  - old values with bxinterval:
+    20.2 μS, total 20 s, 177 MB RAM
+  - rbtree, plain python:
+    97 μS, total 105 s, 846 MB RAM
+  - rbtree converted to cython:
+    26 μS, total 29 s, 320 MB RAM
+  - rbtree and interval converted to cython:
+    8.4 μS, total 12 s, 134 MB RAM
 
 Layouts
 -------

nilmdb/RedBlackTree.cc

@@ -0,0 +1,605 @@
+
+
+
+// The RedBlackEntry class is an Abstract Base Class.  This means that no
+// instance of the RedBlackEntry class can exist.  Only classes which
+// inherit from the RedBlackEntry class can exist.  Furthermore any class
+// which inherits from the RedBlackEntry class must define the member
+// function GetKey().  The Print() member function does not have to
+// be defined because a default definition exists.
+//
+// The GetKey() function should return an integer key for that entry.
+// The key for an entry should never change otherwise bad things might occur.
+
+class RedBlackEntry {
+public:
+  RedBlackEntry();
+  virtual ~RedBlackEntry();
+  virtual int GetKey() const = 0;
+  virtual void Print() const;
+};
+
+class RedBlackTreeNode {
+  friend class RedBlackTree;
+public:
+  void Print(RedBlackTreeNode*,
+	     RedBlackTreeNode*) const;
+  RedBlackTreeNode();
+  RedBlackTreeNode(RedBlackEntry *);
+  RedBlackEntry * GetEntry() const;
+  ~RedBlackTreeNode();
+protected:
+  RedBlackEntry * storedEntry;
+  int key;
+  int red; /* if red=0 then the node is black */
+  RedBlackTreeNode * left;
+  RedBlackTreeNode * right;
+  RedBlackTreeNode * parent;
+};
+
+
+
+class RedBlackTree {
+public:
+  RedBlackTree();
+  ~RedBlackTree();
+  void Print() const;
+  RedBlackEntry * DeleteNode(RedBlackTreeNode *);
+  RedBlackTreeNode * Insert(RedBlackEntry *);
+  RedBlackTreeNode * GetPredecessorOf(RedBlackTreeNode *) const;
+  RedBlackTreeNode * GetSuccessorOf(RedBlackTreeNode *) const;
+  RedBlackTreeNode * Search(int key);
+  TemplateStack<RedBlackTreeNode *> * Enumerate(int low, int high) ;
+  void CheckAssumptions() const;
+protected:
+  /*  A sentinel is used for root and for nil.  These sentinels are */
+  /*  created when RedBlackTreeCreate is caled.  root->left should always */
+  /*  point to the node which is the root of the tree.  nil points to a */
+  /*  node which should always be black but has aribtrary children and */
+  /*  parent and no key or info.  The point of using these sentinels is so */
+  /*  that the root and nil nodes do not require special cases in the code */
+  RedBlackTreeNode * root;
+  RedBlackTreeNode * nil;
+  void LeftRotate(RedBlackTreeNode *);
+  void RightRotate(RedBlackTreeNode *);
+  void TreeInsertHelp(RedBlackTreeNode *);
+  void TreePrintHelper(RedBlackTreeNode *) const;
+  void FixUpMaxHigh(RedBlackTreeNode *);
+  void DeleteFixUp(RedBlackTreeNode *);
+};
+
+const int MIN_INT=-MAX_INT;
+
+RedBlackTreeNode::RedBlackTreeNode(){
+};
+
+RedBlackTreeNode::RedBlackTreeNode(RedBlackEntry * newEntry)
+  : storedEntry (newEntry) , key(newEntry->GetKey()) {
+};
+
+RedBlackTreeNode::~RedBlackTreeNode(){
+};
+
+RedBlackEntry * RedBlackTreeNode::GetEntry() const {return storedEntry;}
+
+RedBlackEntry::RedBlackEntry(){
+};
+RedBlackEntry::~RedBlackEntry(){
+};
+void RedBlackEntry::Print() const {
+  cout << "No Print Method defined. Using Default: " << GetKey() << endl;
+}
+
+RedBlackTree::RedBlackTree()
+{
+  nil = new RedBlackTreeNode;
+  nil->left = nil->right = nil->parent = nil;
+  nil->red = 0;
+  nil->key = MIN_INT;
+  nil->storedEntry = NULL;
+
+  root = new RedBlackTreeNode;
+  root->parent = root->left = root->right = nil;
+  root->key = MAX_INT;
+  root->red=0;
+  root->storedEntry = NULL;
+}
+
+/***********************************************************************/
+/*  FUNCTION:  LeftRotate */
+/**/
+/*  INPUTS:  the node to rotate on */
+/**/
+/*  OUTPUT:  None */
+/**/
+/*  Modifies Input: this, x */
+/**/
+/*  EFFECTS:  Rotates as described in _Introduction_To_Algorithms by */
+/*            Cormen, Leiserson, Rivest (Chapter 14).  Basically this */
+/*            makes the parent of x be to the left of x, x the parent of */
+/*            its parent before the rotation and fixes other pointers */
+/*            accordingly.  */
+/***********************************************************************/
+
+void RedBlackTree::LeftRotate(RedBlackTreeNode* x) {
+  RedBlackTreeNode* y;
+
+  /*  I originally wrote this function to use the sentinel for */
+  /*  nil to avoid checking for nil.  However this introduces a */
+  /*  very subtle bug because sometimes this function modifies */
+  /*  the parent pointer of nil.  This can be a problem if a */
+  /*  function which calls LeftRotate also uses the nil sentinel */
+  /*  and expects the nil sentinel's parent pointer to be unchanged */
+  /*  after calling this function.  For example, when DeleteFixUP */
+  /*  calls LeftRotate it expects the parent pointer of nil to be */
+  /*  unchanged. */
+
+  y=x->right;
+  x->right=y->left;
+
+  if (y->left != nil) y->left->parent=x; /* used to use sentinel here */
+  /* and do an unconditional assignment instead of testing for nil */
+
+  y->parent=x->parent;
+
+  /* instead of checking if x->parent is the root as in the book, we */
+  /* count on the root sentinel to implicitly take care of this case */
+  if( x == x->parent->left) {
+    x->parent->left=y;
+  } else {
+    x->parent->right=y;
+  }
+  y->left=x;
+  x->parent=y;
+}
+
+/***********************************************************************/
+/*  FUNCTION:  RighttRotate */
+/**/
+/*  INPUTS:  node to rotate on */
+/**/
+/*  OUTPUT:  None */
+/**/
+/*  Modifies Input?: this, y */
+/**/
+/*  EFFECTS:  Rotates as described in _Introduction_To_Algorithms by */
+/*            Cormen, Leiserson, Rivest (Chapter 14).  Basically this */
+/*            makes the parent of x be to the left of x, x the parent of */
+/*            its parent before the rotation and fixes other pointers */
+/*            accordingly.  */
+/***********************************************************************/
+
+void RedBlackTree::RightRotate(RedBlackTreeNode* y) {
+  RedBlackTreeNode* x;
+
+  /*  I originally wrote this function to use the sentinel for */
+  /*  nil to avoid checking for nil.  However this introduces a */
+  /*  very subtle bug because sometimes this function modifies */
+  /*  the parent pointer of nil.  This can be a problem if a */
+  /*  function which calls LeftRotate also uses the nil sentinel */
+  /*  and expects the nil sentinel's parent pointer to be unchanged */
+  /*  after calling this function.  For example, when DeleteFixUP */
+  /*  calls LeftRotate it expects the parent pointer of nil to be */
+  /*  unchanged. */
+
+  x=y->left;
+  y->left=x->right;
+
+  if (nil != x->right)  x->right->parent=y; /*used to use sentinel here */
+  /* and do an unconditional assignment instead of testing for nil */
+
+  /* instead of checking if x->parent is the root as in the book, we */
+  /* count on the root sentinel to implicitly take care of this case */
+  x->parent=y->parent;
+  if( y == y->parent->left) {
+    y->parent->left=x;
+  } else {
+    y->parent->right=x;
+  }
+  x->right=y;
+  y->parent=x;
+}
+
+/***********************************************************************/
+/*  FUNCTION:  TreeInsertHelp  */
+/**/
+/*  INPUTS:  z is the node to insert */
+/**/
+/*  OUTPUT:  none */
+/**/
+/*  Modifies Input:  this, z */
+/**/
+/*  EFFECTS:  Inserts z into the tree as if it were a regular binary tree */
+/*            using the algorithm described in _Introduction_To_Algorithms_ */
+/*            by Cormen et al.  This funciton is only intended to be called */
+/*            by the Insert function and not by the user */
+/***********************************************************************/
+
+void RedBlackTree::TreeInsertHelp(RedBlackTreeNode* z) {
+  /*  This function should only be called by RedBlackTree::Insert */
+  RedBlackTreeNode* x;
+  RedBlackTreeNode* y;
+
+  z->left=z->right=nil;
+  y=root;
+  x=root->left;
+  while( x != nil) {
+    y=x;
+    if ( x->key > z->key) {
+      x=x->left;
+    } else { /* x->key <= z->key */
+      x=x->right;
+    }
+  }
+  z->parent=y;
+  if ( (y == root) ||
+       (y->key > z->key) ) {
+    y->left=z;
+  } else {
+    y->right=z;
+  }
+}
+
+/*  Before calling InsertNode  the node x should have its key set */
+
+/***********************************************************************/
+/*  FUNCTION:  InsertNode */
+/**/
+/*  INPUTS:  newEntry is the entry to insert*/
+/**/
+/*  OUTPUT:  This function returns a pointer to the newly inserted node */
+/*           which is guarunteed to be valid until this node is deleted. */
+/*           What this means is if another data structure stores this */
+/*           pointer then the tree does not need to be searched when this */
+/*           is to be deleted. */
+/**/
+/*  Modifies Input: tree */
+/**/
+/*  EFFECTS:  Creates a node node which contains the appropriate key and */
+/*            info pointers and inserts it into the tree. */
+/***********************************************************************/
+/* jim */
+RedBlackTreeNode * RedBlackTree::Insert(RedBlackEntry * newEntry)
+{
+  RedBlackTreeNode * y;
+  RedBlackTreeNode * x;
+  RedBlackTreeNode * newNode;
+
+  x = new RedBlackTreeNode(newEntry);
+  TreeInsertHelp(x);
+  newNode = x;
+  x->red=1;
+  while(x->parent->red) { /* use sentinel instead of checking for root */
+    if (x->parent == x->parent->parent->left) {
+      y=x->parent->parent->right;
+      if (y->red) {
+	x->parent->red=0;
+	y->red=0;
+	x->parent->parent->red=1;
+	x=x->parent->parent;
+      } else {
+	if (x == x->parent->right) {
+	  x=x->parent;
+	  LeftRotate(x);
+	}
+	x->parent->red=0;
+	x->parent->parent->red=1;
+	RightRotate(x->parent->parent);
+      }
+    } else { /* case for x->parent == x->parent->parent->right */
+             /* this part is just like the section above with */
+             /* left and right interchanged */
+      y=x->parent->parent->left;
+      if (y->red) {
+	x->parent->red=0;
+	y->red=0;
+	x->parent->parent->red=1;
+	x=x->parent->parent;
+      } else {
+	if (x == x->parent->left) {
+	  x=x->parent;
+	  RightRotate(x);
+	}
+	x->parent->red=0;
+	x->parent->parent->red=1;
+	LeftRotate(x->parent->parent);
+      }
+    }
+  }
+  root->left->red=0;
+  return(newNode);
+}
+
+/***********************************************************************/
+/*  FUNCTION:  GetSuccessorOf  */
+/**/
+/*    INPUTS:  x is the node we want the succesor of */
+/**/
+/*    OUTPUT:  This function returns the successor of x or NULL if no */
+/*             successor exists. */
+/**/
+/*    Modifies Input: none */
+/**/
+/*    Note:  uses the algorithm in _Introduction_To_Algorithms_ */
+/***********************************************************************/
+
+RedBlackTreeNode * RedBlackTree::GetSuccessorOf(RedBlackTreeNode * x) const
+{
+  RedBlackTreeNode* y;
+
+  if (nil != (y = x->right)) { /* assignment to y is intentional */
+    while(y->left != nil) { /* returns the minium of the right subtree of x */
+      y=y->left;
+    }
+    return(y);
+  } else {
+    y=x->parent;
+    while(x == y->right) { /* sentinel used instead of checking for nil */
+      x=y;
+      y=y->parent;
+    }
+    if (y == root) return(nil);
+    return(y);
+  }
+}
+
+/***********************************************************************/
+/*  FUNCTION:  GetPredecessorOf  */
+/**/
+/*    INPUTS:  x is the node to get predecessor of */
+/**/
+/*    OUTPUT:  This function returns the predecessor of x or NULL if no */
+/*             predecessor exists. */
+/**/
+/*    Modifies Input: none */
+/**/
+/*    Note:  uses the algorithm in _Introduction_To_Algorithms_ */
+/***********************************************************************/
+
+RedBlackTreeNode * RedBlackTree::GetPredecessorOf(RedBlackTreeNode * x) const {
+  RedBlackTreeNode* y;
+
+  if (nil != (y = x->left)) { /* assignment to y is intentional */
+    while(y->right != nil) { /* returns the maximum of the left subtree of x */
+      y=y->right;
+    }
+    return(y);
+  } else {
+    y=x->parent;
+    while(x == y->left) {
+      if (y == root) return(nil);
+      x=y;
+      y=y->parent;
+    }
+    return(y);
+  }
+}
+
+/***********************************************************************/
+/*  FUNCTION:  Print */
+/**/
+/*    INPUTS:  none */
+/**/
+/*    OUTPUT:  none  */
+/**/
+/*    EFFECTS:  This function recursively prints the nodes of the tree */
+/*              inorder. */
+/**/
+/*    Modifies Input: none */
+/**/
+/*    Note:    This function should only be called from ITTreePrint */
+/***********************************************************************/
+
+void RedBlackTreeNode::Print(RedBlackTreeNode * nil,
+			     RedBlackTreeNode * root) const {
+  storedEntry->Print();
+  printf(", key=%i ",key);
+  printf("  l->key=");
+  if( left == nil) printf("NULL"); else printf("%i",left->key);
+  printf("  r->key=");
+  if( right == nil) printf("NULL"); else printf("%i",right->key);
+  printf("  p->key=");
+  if( parent == root) printf("NULL"); else printf("%i",parent->key);
+  printf("  red=%i\n",red);
+}
+
+void RedBlackTree::TreePrintHelper( RedBlackTreeNode* x) const {
+
+  if (x != nil) {
+    TreePrintHelper(x->left);
+    x->Print(nil,root);
+    TreePrintHelper(x->right);
+  }
+}
+
+/***********************************************************************/
+/*  FUNCTION:  Print */
+/**/
+/*    INPUTS:  none */
+/**/
+/*    OUTPUT:  none */
+/**/
+/*    EFFECT:  This function recursively prints the nodes of the tree */
+/*             inorder. */
+/**/
+/*    Modifies Input: none */
+/**/
+/***********************************************************************/
+
+void RedBlackTree::Print() const {
+  TreePrintHelper(root->left);
+}
+
+/***********************************************************************/
+/*  FUNCTION:  DeleteFixUp */
+/**/
+/*    INPUTS:  x is the child of the spliced */
+/*             out node in DeleteNode. */
+/**/
+/*    OUTPUT:  none */
+/**/
+/*    EFFECT:  Performs rotations and changes colors to restore red-black */
+/*             properties after a node is deleted */
+/**/
+/*    Modifies Input: this, x */
+/**/
+/*    The algorithm from this function is from _Introduction_To_Algorithms_ */
+/***********************************************************************/
+
+void RedBlackTree::DeleteFixUp(RedBlackTreeNode* x) {
+  RedBlackTreeNode * w;
+  RedBlackTreeNode * rootLeft = root->left;
+
+  while( (!x->red) && (rootLeft != x)) {
+    if (x == x->parent->left) {
+
+//
+      w=x->parent->right;
+      if (w->red) {
+	w->red=0;
+	x->parent->red=1;
+	LeftRotate(x->parent);
+	w=x->parent->right;
+      }
+      if ( (!w->right->red) && (!w->left->red) ) {
+	w->red=1;
+	x=x->parent;
+      } else {
+	if (!w->right->red) {
+	  w->left->red=0;
+	  w->red=1;
+	  RightRotate(w);
+	  w=x->parent->right;
+	}
+	w->red=x->parent->red;
+	x->parent->red=0;
+	w->right->red=0;
+	LeftRotate(x->parent);
+	x=rootLeft; /* this is to exit while loop */
+      }
+//
+
+    } else { /* the code below is has left and right switched from above */
+      w=x->parent->left;
+      if (w->red) {
+	w->red=0;
+	x->parent->red=1;
+	RightRotate(x->parent);
+	w=x->parent->left;
+      }
+      if ( (!w->right->red) && (!w->left->red) ) {
+	w->red=1;
+	x=x->parent;
+      } else {
+	if (!w->left->red) {
+	  w->right->red=0;
+	  w->red=1;
+	  LeftRotate(w);
+	  w=x->parent->left;
+	}
+	w->red=x->parent->red;
+	x->parent->red=0;
+	w->left->red=0;
+	RightRotate(x->parent);
+	x=rootLeft; /* this is to exit while loop */
+      }
+    }
+  }
+  x->red=0;
+
+}
+
+
+/***********************************************************************/
+/*  FUNCTION:  DeleteNode */
+/**/
+/*    INPUTS:  tree is the tree to delete node z from */
+/**/
+/*    OUTPUT:  returns the RedBlackEntry stored at deleted node */
+/**/
+/*    EFFECT:  Deletes z from tree and but don't call destructor */
+/**/
+/*    Modifies Input:  z */
+/**/
+/*    The algorithm from this function is from _Introduction_To_Algorithms_ */
+/***********************************************************************/
+
+RedBlackEntry * RedBlackTree::DeleteNode(RedBlackTreeNode * z){
+  RedBlackTreeNode* y;
+  RedBlackTreeNode* x;
+  RedBlackEntry * returnValue = z->storedEntry;
+
+  y= ((z->left == nil) || (z->right == nil)) ? z : GetSuccessorOf(z);
+  x= (y->left == nil) ? y->right : y->left;
+  if (root == (x->parent = y->parent)) { /* assignment of y->p to x->p is intentional */
+    root->left=x;
+  } else {
+    if (y == y->parent->left) {
+      y->parent->left=x;
+    } else {
+      y->parent->right=x;
+    }
+  }
+  if (y != z) { /* y should not be nil in this case */
+
+    /* y is the node to splice out and x is its child */
+
+    y->left=z->left;
+    y->right=z->right;
+    y->parent=z->parent;
+    z->left->parent=z->right->parent=y;
+    if (z == z->parent->left) {
+      z->parent->left=y;
+    } else {
+      z->parent->right=y;
+    }
+    if (!(y->red)) {
+      y->red = z->red;
+      DeleteFixUp(x);
+    } else
+      y->red = z->red;
+    delete z;
+  } else {
+    if (!(y->red)) DeleteFixUp(x);
+    delete y;
+  }
+  return returnValue;
+}
+
+
+/***********************************************************************/
+/*  FUNCTION:  Enumerate */
+/**/
+/*    INPUTS:  tree is the tree to look for keys between [low,high] */
+/**/
+/*    OUTPUT:  stack containing pointers to the nodes between [low,high] */
+/**/
+/*    Modifies Input: none */
+/**/
+/*    EFFECT:  Returns a stack containing pointers to nodes containing */
+/*             keys which in [low,high]/ */
+/**/
+/***********************************************************************/
+
+TemplateStack<RedBlackTreeNode *> * RedBlackTree::Enumerate(int low,
+							    int high)  {
+  TemplateStack<RedBlackTreeNode *> * enumResultStack =
+    new TemplateStack<RedBlackTreeNode *>(4);
+
+  RedBlackTreeNode* x=root->left;
+  RedBlackTreeNode* lastBest=NULL;
+
+  while(nil != x) {
+    if ( x->key > high ) {
+      x=x->left;
+    } else {
+      lastBest=x;
+      x=x->right;
+    }
+  }
+  while ( (lastBest) && (low <= lastBest->key) ) {
+    enumResultStack->Push(lastBest);
+    lastBest=GetPredecessorOf(lastBest);
+  }
+  return(enumResultStack);
+}

nilmdb/client.py

@@ -84,6 +84,11 @@ class Client(object):
                    "layout" : layout }
         return self.http.get("stream/create", params)
 
+    def stream_destroy(self, path):
+        """Delete stream and its contents"""
+        params = { "path": path }
+        return self.http.get("stream/destroy", params)
+
     def stream_insert(self, path, data):
         """Insert data into a stream.  data should be a file-like object
         that provides ASCII data that matches the database layout for path."""

nilmdb/cmdline/cmdline.py

@@ -15,7 +15,8 @@ version = "0.1"
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract" ]
+subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
+                "destroy" ]
 
 # Import the subcommand modules.  Equivalent way of doing this would be
 # from . import info as cmd_info

nilmdb/cmdline/destroy.py

@@ -0,0 +1,25 @@
+from __future__ import absolute_import
+from nilmdb.printf import *
+import nilmdb.client
+
+from argparse import ArgumentDefaultsHelpFormatter as def_form
+
+def setup(self, sub):
+    cmd = sub.add_parser("destroy", help="Delete a stream and all data",
+                         formatter_class = def_form,
+                         description="""
+                         Destroy the stream at the specified path.  All
+                         data and metadata related to the stream is
+                         permanently deleted.
+                         """)
+    cmd.set_defaults(handler = cmd_destroy)
+    group = cmd.add_argument_group("Required arguments")
+    group.add_argument("path",
+                       help="Path of the stream to delete, e.g. /foo/bar")
+
+def cmd_destroy(self):
+    """Destroy stream"""
+    try:
+        self.client.stream_destroy(self.args.path)
+    except nilmdb.client.ClientError as e:
+        self.die("Error deleting stream: %s", str(e))

nilmdb/interval.pyx

@@ -1,58 +1,81 @@
-"""Interval and IntervalSet
+"""Interval, IntervalSet
 
 Represents an interval of time, and a set of such intervals.
 
-Intervals are closed, ie. they include timestamps [start, end]
+Intervals are half-open, ie. they include data points with timestamps
+[start, end)
 """
 
 # First implementation kept a sorted list of intervals and used
 # biesct() to optimize some operations, but this was too slow.
 
-# This version is based on the quicksect implementation from python-bx,
-# modified slightly to handle floating point intervals.
+# Second version was based on the quicksect implementation from
+# python-bx, modified slightly to handle floating point intervals.
+# This didn't support deletion.
 
-import pyximport
-pyximport.install()
-import bxintersect
+# Third version is more similar to the first version, using a rb-tree
+# instead of a simple sorted list to maintain O(log n) operations.
 
-import bisect
+# Fourth version is an optimized rb-tree that stores interval starts
+# and ends directly in the tree, like bxinterval did.
+
+cimport rbtree
+cdef extern from "stdint.h":
+    ctypedef unsigned long long uint64_t
 
 class IntervalError(Exception):
     """Error due to interval overlap, etc"""
     pass
 
-class Interval(bxintersect.Interval):
+cdef class Interval:
     """Represents an interval of time."""
 
-    def __init__(self, start, end):
+    cdef public double start, end
+
+    def __init__(self, double start, double end):
         """
         'start' and 'end' are arbitrary floats that represent time
         """
         if start > end:
             raise IntervalError("start %s must precede end %s" % (start, end))
-        bxintersect.Interval.__init__(self, start, end)
+        self.start = float(start)
+        self.end = float(end)
 
     def __repr__(self):
         s = repr(self.start) + ", " + repr(self.end)
         return self.__class__.__name__ + "(" + s + ")"
 
     def __str__(self):
-        return "[" + str(self.start) + " -> " + str(self.end) + "]"
+        return "[" + repr(self.start) + " -> " + repr(self.end) + ")"
 
-    def intersects(self, other):
+    def __cmp__(self, Interval other):
+        """Compare two intervals.  If non-equal, order by start then end"""
+        if not isinstance(other, Interval):
+            raise TypeError("bad type")
+        if self.start == other.start:
+            if self.end < other.end:
+                return -1
+            if self.end > other.end:
+                return 1
+            return 0
+        if self.start < other.start:
+            return -1
+        return 1
+
+    cpdef intersects(self, Interval other):
         """Return True if two Interval objects intersect"""
         if (self.end <= other.start or self.start >= other.end):
             return False
         return True
 
-    def subset(self, start, end):
+    cpdef subset(self, double start, double end):
         """Return a new Interval that is a subset of this one"""
         # A subclass that tracks additional data might override this.
         if start < self.start or end > self.end:
             raise IntervalError("not a subset")
         return Interval(start, end)
 
-class DBInterval(Interval):
+cdef class DBInterval(Interval):
     """
     Like Interval, but also tracks corresponding start/end times and
     positions within the database.  These are not currently modified
@@ -66,6 +89,10 @@ class DBInterval(Interval):
         end = 150
         db_end = 200, db_endpos = 20000
     """
+
+    cpdef public double db_start, db_end
+    cpdef public uint64_t db_startpos, db_endpos
+
     def __init__(self, start, end,
                  db_start, db_end,
                  db_startpos, db_endpos):
@@ -90,7 +117,7 @@ class DBInterval(Interval):
         s += ", " + repr(self.db_startpos) + ", " + repr(self.db_endpos)
         return self.__class__.__name__ + "(" + s + ")"
 
-    def subset(self, start, end):
+    cpdef subset(self, double start, double end):
         """
         Return a new DBInterval that is a subset of this one
         """
@@ -100,21 +127,25 @@ class DBInterval(Interval):
                           self.db_start, self.db_end,
                           self.db_startpos, self.db_endpos)
 
-class IntervalSet(object):
+cdef class IntervalSet:
     """
     A non-intersecting set of intervals.
     """
 
+    cdef public rbtree.RBTree tree
+
     def __init__(self, source=None):
         """
         'source' is an Interval or IntervalSet to add.
         """
-        self.tree = bxintersect.IntervalTree()
+        self.tree = rbtree.RBTree()
         if source is not None:
             self += source
 
     def __iter__(self):
-        return self.tree.traverse()
+        for node in self.tree:
+            if node.obj:
+                yield node.obj
 
     def __len__(self):
         return sum(1 for x in self)
@@ -127,7 +158,7 @@ class IntervalSet(object):
         descs = [ str(x) for x in self ]
         return  "[" + ", ".join(descs) + "]"
 
-    def __eq__(self, other):
+    def __match__(self, other):
         # This isn't particularly efficient, but it shouldn't get used in the
         # general case.
         """Test equality of two IntervalSets.
@@ -178,10 +209,20 @@ class IntervalSet(object):
         except IndexError:
             return False
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
+    # Use __richcmp__ instead of __eq__, __ne__ for Cython.
+    def __richcmp__(self, other, int op):
+        if op == 2: # ==
+            return self.__match__(other)
+        elif op == 3: # !=
+            return not self.__match__(other)
+        return False
+    #def __eq__(self, other):
+    #    return self.__match__(other)
+    #
+    #def __ne__(self, other):
+    #    return not self.__match__(other)
 
-    def __iadd__(self, other):
+    def __iadd__(self, object other not None):
         """Inplace add -- modifies self
 
         This throws an exception if the regions being added intersect."""
@@ -189,19 +230,30 @@ class IntervalSet(object):
             if self.intersects(other):
                 raise IntervalError("Tried to add overlapping interval "
                                     "to this set")
-            self.tree.insert_interval(other)
+            self.tree.insert(rbtree.RBNode(other.start, other.end, other))
         else:
             for x in other:
                 self.__iadd__(x)
         return self
 
-    def __add__(self, other):
+    def __isub__(self, Interval other not None):
+        """Inplace subtract -- modifies self
+
+        Removes an interval from the set.  Must exist exactly
+        as provided -- cannot remove a subset of an existing interval."""
+        i = self.tree.find(other.start, other.end)
+        if i is None:
+            raise IntervalError("interval " + str(other) + " not in tree")
+        self.tree.delete(i)
+        return self
+
+    def __add__(self, other not None):
         """Add -- returns a new object"""
         new = IntervalSet(self)
         new += IntervalSet(other)
         return new
 
-    def __and__(self, other):
+    def __and__(self, other not None):
         """
         Compute a new IntervalSet from the intersection of two others
 
@@ -211,15 +263,16 @@ class IntervalSet(object):
         out = IntervalSet()
 
         if not isinstance(other, IntervalSet):
-            other = [ other ]
-
-        for x in other:
-            for i in self.intersection(x):
-                out.tree.insert_interval(i)
+            for i in self.intersection(other):
+                out.tree.insert(rbtree.RBNode(i.start, i.end, i))
+        else:
+            for x in other:
+                for i in self.intersection(x):
+                    out.tree.insert(rbtree.RBNode(i.start, i.end, i))
 
         return out
 
-    def intersection(self, interval):
+    def intersection(self, Interval interval not None):
         """
         Compute a sequence of intervals that correspond to the
         intersection between `self` and the provided interval.
@@ -229,13 +282,21 @@ class IntervalSet(object):
         Output intervals are built as subsets of the intervals in the
         first argument (self).
         """
-        for i in self.tree.find(interval.start, interval.end):
-            if i.start > interval.start and i.end < interval.end:
-                yield i
-            else:
-                yield i.subset(max(i.start, interval.start),
-                               min(i.end, interval.end))
+        if not isinstance(interval, Interval):
+            raise TypeError("bad type")
+        for n in self.tree.intersect(interval.start, interval.end):
+            i = n.obj
+            if i:
+                if i.start >= interval.start and i.end <= interval.end:
+                    yield i
+                else:
+                    subset = i.subset(max(i.start, interval.start),
+                                      min(i.end, interval.end))
+                    yield subset
 
-    def intersects(self, other):
+    cpdef intersects(self, Interval other):
         """Return True if this IntervalSet intersects another interval"""
-        return len(self.tree.find(other.start, other.end)) > 0
+        for n in self.tree.intersect(other.start, other.end):
+            if n.obj.intersects(other):
+                return True
+        return False

nilmdb/interval.pyxdep

@@ -0,0 +1 @@
+rbtree.pxd

nilmdb/nilmdb.py

@@ -291,16 +291,6 @@ class NilmDB(object):
         if group == '':
             raise ValueError("invalid path")
 
-        # Make the group structure, one element at a time
-        group_path = group.lstrip('/').split("/")
-        for i in range(len(group_path)):
-            parent = "/" + "/".join(group_path[0:i])
-            child = group_path[i]
-            try:
-                self.h5file.createGroup(parent, child)
-            except tables.NodeError:
-                pass
-
         # Get description
         try:
             desc = nilmdb.layout.get_named(layout_name).description()
@@ -312,9 +302,15 @@ class NilmDB(object):
         exp_rows = 8000 * 60*60*24*30*3
 
         # Create the table
-        table = self.h5file.createTable(group, node,
-                                        description = desc,
-                                        expectedrows = exp_rows)
+        try:
+            table = self.h5file.createTable(group, node,
+                                            description = desc,
+                                            expectedrows = exp_rows,
+                                            createparents = True)
+        except AttributeError:
+            # Trying to create e.g. /foo/bar/baz when /foo/bar is already
+            # a table raises this error.
+            raise ValueError("error creating table at that path")
 
         # Insert into SQL database once the PyTables is happy
         with self.con as con:
@@ -337,8 +333,7 @@ class NilmDB(object):
            """
         stream_id = self._stream_id(path)
         with self.con as con:
-            con.execute("DELETE FROM metadata "
-                        "WHERE stream_id=?", (stream_id,))
+            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
             for key in data:
                 if data[key] != '':
                     con.execute("INSERT INTO metadata VALUES (?, ?, ?)",
@@ -361,6 +356,33 @@ class NilmDB(object):
         data.update(newdata)
         self.stream_set_metadata(path, data)
 
+    def stream_destroy(self, path):
+        """Fully remove a table and all of its data from the database.
+        No way to undo it!  The group structure is removed, if there
+        are no other tables in it.  Metadata is removed."""
+        stream_id = self._stream_id(path)
+
+        # Delete the cached interval data
+        if stream_id in self._cached_iset:
+            del self._cached_iset[stream_id]
+
+        # Delete the data node, and all parent nodes (if they have no
+        # remaining children)
+        split_path = path.lstrip('/').split("/")
+        while split_path:
+            name = split_path.pop()
+            where = "/" + "/".join(split_path)
+            try:
+                self.h5file.removeNode(where, name, recursive = False)
+            except tables.NodeError:
+                break
+
+        # Delete metadata, stream, intervals
+        with self.con as con:
+            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
+            con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
+            con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
+
     def stream_insert(self, path, parser, old_timestamp = None):
         """Insert new data into the database.
            path: Path at which to add the data
@@ -386,7 +408,7 @@ class NilmDB(object):
         iset = self._get_intervals(stream_id)
         interval = Interval(min_timestamp, parser.max_timestamp)
         if iset.intersects(interval):
-            raise OverlapError("new data overlaps existing data: "
+            raise OverlapError("new data overlaps existing data at range: "
                                + str(iset & interval))
 
         # Insert the data into pytables

nilmdb/rbtree.pxd

@@ -0,0 +1,23 @@
+cdef class RBNode:
+    cdef public object obj
+    cdef public double start, end
+    cdef public int red
+    cdef public RBNode left, right, parent
+
+cdef class RBTree:
+    cdef public RBNode nil, root
+
+    cpdef getroot(RBTree self)
+    cdef void __rotate_left(RBTree self, RBNode x)
+    cdef void __rotate_right(RBTree self, RBNode y)
+    cdef RBNode __successor(RBTree self, RBNode x)
+    cpdef RBNode successor(RBTree self, RBNode x)
+    cdef RBNode __predecessor(RBTree self, RBNode x)
+    cpdef RBNode predecessor(RBTree self, RBNode x)
+    cpdef insert(RBTree self, RBNode z)
+    cdef void __insert_fixup(RBTree self, RBNode x)
+    cpdef delete(RBTree self, RBNode z)
+    cdef inline void __delete_fixup(RBTree self, RBNode x)
+    cpdef RBNode find(RBTree self, double start, double end)
+    cpdef RBNode find_left_end(RBTree self, double t)
+    cpdef RBNode find_right_start(RBTree self, double t)

nilmdb/rbtree.pyx

@@ -0,0 +1,377 @@
+# cython: profile=False
+# cython: cdivision=True
+
+"""
+Jim Paris <jim@jtan.com>
+
+Red-black tree, where keys are stored as start/end timestamps.
+This is a basic interval tree that holds half-open intervals:
+  [start, end)
+Intervals must not overlap.  Fixing that would involve making this
+into an augmented interval tree as described in CLRS 14.3.
+
+Code that assumes non-overlapping intervals is marked with the
+string 'non-overlapping'.
+"""
+
+import sys
+cimport rbtree
+
+cdef class RBNode:
+    """One node of the Red/Black tree, containing a key (start, end)
+    and value (obj)"""
+    def __init__(self, double start, double end, object obj = None):
+        self.obj = obj
+        self.start = start
+        self.end = end
+        self.red = False
+        self.left = None
+        self.right = None
+
+    def __str__(self):
+        if self.red:
+            color = "R"
+        else:
+            color = "B"
+        if self.start == sys.float_info.min:
+            return "[node nil]"
+        return ("[node ("
+                + str(self.obj) + ") "
+                + str(self.start) + " -> " + str(self.end) + " "
+                + color + "]")
+
+cdef class RBTree:
+    """Red/Black tree"""
+
+    # Init
+    def __init__(self):
+        self.nil = RBNode(start = sys.float_info.min,
+                          end = sys.float_info.min)
+        self.nil.left = self.nil
+        self.nil.right = self.nil
+        self.nil.parent = self.nil
+
+        self.root = RBNode(start = sys.float_info.max,
+                           end = sys.float_info.max)
+        self.root.left = self.nil
+        self.root.right = self.nil
+        self.root.parent = self.nil
+
+    # We have a dummy root node to simplify operations, so from an
+    # external point of view, its left child is the real root.
+    cpdef getroot(self):
+        return self.root.left
+
+    # Rotations and basic operations
+    cdef void __rotate_left(self, RBNode x):
+        """Rotate left:
+        #   x           y
+        #  / \   -->   / \
+        # z   y       x   w
+        #    / \     / \
+        #   v   w   z   v
+        """
+        cdef RBNode y = x.right
+        x.right = y.left
+        if y.left is not self.nil:
+            y.left.parent = x
+        y.parent = x.parent
+        if x is x.parent.left:
+            x.parent.left = y
+        else:
+            x.parent.right = y
+        y.left = x
+        x.parent = y
+
+    cdef void __rotate_right(self, RBNode y):
+        """Rotate right:
+        #     y           x
+        #    / \   -->   / \
+        #   x   w       z   y
+        #  / \             / \
+        # z   v           v   w
+        """
+        cdef RBNode x = y.left
+        y.left = x.right
+        if x.right is not self.nil:
+            x.right.parent = y
+        x.parent = y.parent
+        if y is y.parent.left:
+            y.parent.left = x
+        else:
+            y.parent.right = x
+        x.right = y
+        y.parent = x
+
+    cdef RBNode __successor(self, RBNode x):
+        """Returns the successor of RBNode x"""
+        cdef RBNode y = x.right
+        if y is not self.nil:
+            while y.left is not self.nil:
+                y = y.left
+        else:
+            y = x.parent
+            while x is y.right:
+                x = y
+                y = y.parent
+            if y is self.root:
+                return self.nil
+        return y
+    cpdef RBNode successor(self, RBNode x):
+        """Returns the successor of RBNode x, or None"""
+        cdef RBNode y = self.__successor(x)
+        return y if y is not self.nil else None
+
+    cdef RBNode __predecessor(self, RBNode x):
+        """Returns the predecessor of RBNode x"""
+        cdef RBNode y = x.left
+        if y is not self.nil:
+            while y.right is not self.nil:
+                y = y.right
+        else:
+            y = x.parent
+            while x is y.left:
+                if y is self.root:
+                    y = self.nil
+                    break
+                x = y
+                y = y.parent
+        return y
+    cpdef RBNode predecessor(self, RBNode x):
+        """Returns the predecessor of RBNode x, or None"""
+        cdef RBNode y = self.__predecessor(x)
+        return y if y is not self.nil else None
+
+    # Insertion
+    cpdef insert(self, RBNode z):
+        """Insert RBNode z into RBTree and rebalance as necessary"""
+        z.left = self.nil
+        z.right = self.nil
+        cdef RBNode y = self.root
+        cdef RBNode x = self.root.left
+        while x is not self.nil:
+            y = x
+            if (x.start > z.start or (x.start == z.start and x.end > z.end)):
+                x = x.left
+            else:
+                x = x.right
+        z.parent = y
+        if (y is self.root or
+            (y.start > z.start or (y.start == z.start and y.end > z.end))):
+            y.left = z
+        else:
+            y.right = z
+        # relabel/rebalance
+        self.__insert_fixup(z)
+
+    cdef void __insert_fixup(self, RBNode x):
+        """Rebalance/fix RBTree after a simple insertion of RBNode x"""
+        x.red = True
+        while x.parent.red:
+            if x.parent is x.parent.parent.left:
+                y = x.parent.parent.right
+                if y.red:
+                    x.parent.red = False
+                    y.red = False
+                    x.parent.parent.red = True
+                    x = x.parent.parent
+                else:
+                    if x is x.parent.right:
+                        x = x.parent
+                        self.__rotate_left(x)
+                    x.parent.red = False
+                    x.parent.parent.red = True
+                    self.__rotate_right(x.parent.parent)
+            else: # same as above, left/right switched
+                y = x.parent.parent.left
+                if y.red:
+                    x.parent.red = False
+                    y.red = False
+                    x.parent.parent.red = True
+                    x = x.parent.parent
+                else:
+                    if x is x.parent.left:
+                        x = x.parent
+                        self.__rotate_right(x)
+                    x.parent.red = False
+                    x.parent.parent.red = True
+                    self.__rotate_left(x.parent.parent)
+        self.root.left.red = False
+
+    # Deletion
+    cpdef delete(self, RBNode z):
+        if z.left is None or z.right is None:
+            raise AttributeError("you can only delete a node object "
+                                 + "from the tree; use find() to get one")
+        cdef RBNode x, y
+        if z.left is self.nil or z.right is self.nil:
+            y = z
+        else:
+            y = self.__successor(z)
+        if y.left is self.nil:
+            x = y.right
+        else:
+            x = y.left
+        x.parent = y.parent
+        if x.parent is self.root:
+            self.root.left = x
+        else:
+            if y is y.parent.left:
+                y.parent.left = x
+            else:
+                y.parent.right = x
+        if y is not z:
+            # y is the node to splice out, x is its child
+            y.left = z.left
+            y.right = z.right
+            y.parent = z.parent
+            z.left.parent = y
+            z.right.parent = y
+            if z is z.parent.left:
+                z.parent.left = y
+            else:
+                z.parent.right = y
+            if not y.red:
+                y.red = z.red
+                self.__delete_fixup(x)
+            else:
+                y.red = z.red
+        else:
+            if not y.red:
+                self.__delete_fixup(x)
+
+    cdef void __delete_fixup(self, RBNode x):
+        """Rebalance/fix RBTree after a deletion.  RBNode x is the
+        child of the spliced out node."""
+        cdef RBNode rootLeft = self.root.left
+        while not x.red and x is not rootLeft:
+            if x is x.parent.left:
+                w = x.parent.right
+                if w.red:
+                    w.red = False
+                    x.parent.red = True
+                    self.__rotate_left(x.parent)
+                    w = x.parent.right
+                if not w.right.red and not w.left.red:
+                    w.red = True
+                    x = x.parent
+                else:
+                    if not w.right.red:
+                        w.left.red = False
+                        w.red = True
+                        self.__rotate_right(w)
+                        w = x.parent.right
+                    w.red = x.parent.red
+                    x.parent.red = False
+                    w.right.red = False
+                    self.__rotate_left(x.parent)
+                    x = rootLeft # exit loop
+            else: # same as above, left/right switched
+                w = x.parent.left
+                if w.red:
+                    w.red = False
+                    x.parent.red = True
+                    self.__rotate_right(x.parent)
+                    w = x.parent.left
+                if not w.left.red and not w.right.red:
+                    w.red = True
+                    x = x.parent
+                else:
+                    if not w.left.red:
+                        w.right.red = False
+                        w.red = True
+                        self.__rotate_left(w)
+                        w = x.parent.left
+                    w.red = x.parent.red
+                    x.parent.red = False
+                    w.left.red = False
+                    self.__rotate_right(x.parent)
+                    x = rootLeft # exit loop
+        x.red = False
+
+    # Walking, searching
+    def __iter__(self):
+        return self.inorder()
+
+    def inorder(self, RBNode x = None):
+        """Generator that performs an inorder walk for the tree
+        rooted at RBNode x"""
+        if x is None:
+            x = self.getroot()
+        while x.left is not self.nil:
+            x = x.left
+        while x is not self.nil:
+            yield x
+            x = self.__successor(x)
+
+    cpdef RBNode find(self, double start, double end):
+        """Return the node with exactly the given start and end."""
+        cdef RBNode x = self.getroot()
+        while x is not self.nil:
+            if start < x.start:
+                x = x.left
+            elif start == x.start:
+                if end == x.end:
+                    break # found it
+                elif end < x.end:
+                    x = x.left
+                else:
+                    x = x.right
+            else:
+                x = x.right
+        return x if x is not self.nil else None
+
+    cpdef RBNode find_left_end(self, double t):
+        """Find the leftmode node with end >= t.  With non-overlapping
+        intervals, this is the first node that might overlap time t.
+
+        Note that this relies on non-overlapping intervals, since
+        it assumes that we can use the endpoints to traverse the
+        tree even though it was created using the start points."""
+        cdef RBNode x = self.getroot()
+        while x is not self.nil:
+            if t < x.end:
+                if x.left is self.nil:
+                    break
+                x = x.left
+            elif t == x.end:
+                break
+            else:
+                if x.right is self.nil:
+                    x = self.__successor(x)
+                    break
+                x = x.right
+        return x if x is not self.nil else None
+
+    cpdef RBNode find_right_start(self, double t):
+        """Find the rightmode node with start <= t.  With non-overlapping
+        intervals, this is the last node that might overlap time t."""
+        cdef RBNode x = self.getroot()
+        while x is not self.nil:
+            if t < x.start:
+                if x.left is self.nil:
+                    x = self.__predecessor(x)
+                    break
+                x = x.left
+            elif t == x.start:
+                break
+            else:
+                if x.right is self.nil:
+                    break
+                x = x.right
+        return x if x is not self.nil else None
+
+    # Intersections
+    def intersect(self, double start, double end):
+        """Generator that returns nodes that overlap the given
+        (start,end) range.  Assumes non-overlapping intervals."""
+        # Start with the leftmode node that ends after start
+        cdef RBNode n = self.find_left_end(start)
+        while n is not None:
+            if n.start >= end:
+                # this node starts after the requested end; we're done
+                break
+            if start < n.end:
+                # this node overlaps our requested area
+                yield n
+            n = self.successor(n)

nilmdb/rbtree.pyxdep

@@ -0,0 +1 @@
+rbtree.pxd

nilmdb/server.py

@@ -88,6 +88,17 @@ class Stream(NilmApp):
             message = sprintf("%s: %s", type(e).__name__, e.message)
             raise cherrypy.HTTPError("400 Bad Request", message)
 
+    # /stream/destroy?path=/newton/prep
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    def destroy(self, path):
+        """Delete a stream and its associated data."""
+        try:
+            return self.db.stream_destroy(path)
+        except Exception as e:
+            message = sprintf("%s: %s", type(e).__name__, e.message)
+            raise cherrypy.HTTPError("400 Bad Request", message)
+
     # /stream/get_metadata?path=/newton/prep
     # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
     @cherrypy.expose

setup.cfg

@@ -12,7 +12,10 @@ stop=
 verbosity=2
 #tests=tests/test_cmdline.py
 #tests=tests/test_layout.py
-tests=tests/test_interval.py
+#tests=tests/test_rbtree.py
+#tests=tests/test_interval.py
+#tests=tests/test_rbtree.py,tests/test_interval.py
+#tests=tests/test_interval.py
 #tests=tests/test_client.py
 #tests=tests/test_timestamper.py
 #tests=tests/test_serializer.py

tests/renderdot.py

@@ -0,0 +1,90 @@
+import sys
+
+class Renderer(object):
+
+    def __init__(self, getleft, getright,
+                 getred, getstart, getend, nil):
+        self.getleft = getleft
+        self.getright = getright
+        self.getred = getred
+        self.getstart = getstart
+        self.getend = getend
+        self.nil = nil
+
+    # Rendering
+    def __render_dot_node(self, node, max_depth = 20):
+        from nilmdb.printf import sprintf
+        """Render a single node and its children into a dot graph fragment"""
+        if max_depth == 0:
+            return ""
+        if node is self.nil:
+            return ""
+        def c(red):
+            if red:
+                return 'color="#ff0000", style=filled, fillcolor="#ffc0c0"'
+            else:
+                return 'color="#000000", style=filled, fillcolor="#c0c0c0"'
+        s = sprintf("%d [label=\"%g\\n%g\", %s];\n",
+                    id(node),
+                    self.getstart(node), self.getend(node),
+                    c(self.getred(node)))
+
+        if self.getleft(node) is self.nil:
+            s += sprintf("L%d [label=\"-\", %s];\n", id(node), c(False))
+            s += sprintf("%d -> L%d [label=L];\n", id(node), id(node))
+        else:
+            s += sprintf("%d -> %d [label=L];\n",
+                         id(node),id(self.getleft(node)))
+        if self.getright(node) is self.nil:
+            s += sprintf("R%d [label=\"-\", %s];\n", id(node), c(False))
+            s += sprintf("%d -> R%d [label=R];\n", id(node), id(node))
+        else:
+            s += sprintf("%d -> %d [label=R];\n",
+                         id(node), id(self.getright(node)))
+        s += self.__render_dot_node(self.getleft(node), max_depth-1)
+        s += self.__render_dot_node(self.getright(node), max_depth-1)
+        return s
+
+    def render_dot(self, rootnode, title = "Tree"):
+        """Render the entire tree as a dot graph"""
+        return ("digraph rbtree {\n"
+                + self.__render_dot_node(rootnode)
+                + "}\n");
+
+    def render_dot_live(self, rootnode, title = "Tree"):
+        """Render the entiretree as a dot graph, live GTK view"""
+        import gtk
+        import gtk.gdk
+        sys.path.append("/usr/share/xdot")
+        import xdot
+        xdot.Pen.highlighted = lambda pen: pen
+        s = ("digraph rbtree {\n"
+             + self.__render_dot_node(rootnode)
+             + "}\n");
+        window = xdot.DotWindow()
+        window.set_dotcode(s)
+        window.set_title(title + " - any key to close")
+        window.connect('destroy', gtk.main_quit)
+        def quit(widget, event):
+            if not event.is_modifier:
+                window.destroy()
+                gtk.main_quit()
+        window.widget.connect('key-press-event', quit)
+        gtk.main()
+
+class RBTreeRenderer(Renderer):
+    def __init__(self, tree):
+        Renderer.__init__(self,
+                          lambda node: node.left,
+                          lambda node: node.right,
+                          lambda node: node.red,
+                          lambda node: node.start,
+                          lambda node: node.end,
+                          tree.nil)
+        self.tree = tree
+
+    def render(self, title = "RBTree", live = True):
+        if live:
+            return Renderer.render_dot_live(self, self.tree.getroot(), title)
+        else:
+            return Renderer.render_dot(self, self.tree.getroot(), title)

tests/test_cmdline.py

@@ -192,11 +192,17 @@ class TestCmdline(object):
         self.contain("no such layout")
 
         # Create a few streams
+        self.ok("create /newton/zzz/rawnotch RawNotchedData")
         self.ok("create /newton/prep PrepData")
         self.ok("create /newton/raw RawData")
-        self.ok("create /newton/zzz/rawnotch RawNotchedData")
 
-        # Verify we got those 3 streams
+        # Should not be able to create a stream with another stream as
+        # its parent
+        self.fail("create /newton/prep/blah PrepData")
+        self.contain("error creating table at that path")
+
+        # Verify we got those 3 streams and they're returned in
+        # alphabetical order.
         self.ok("list")
         self.match("/newton/prep PrepData\n"
                    "/newton/raw RawData\n"
@@ -456,3 +462,54 @@ class TestCmdline(object):
         eq_(self.captured.count('\n'), 11)
         server_stop()
         server_start()
+
+    def test_cmdline_10_destroy(self):
+        # Delete records
+        self.ok("destroy --help")
+
+        self.fail("destroy")
+        self.contain("too few arguments")
+
+        self.fail("destroy /no/such/stream")
+        self.contain("No stream at path")
+
+        self.fail("destroy asdfasdf")
+        self.contain("No stream at path")
+
+        # From previous tests, we have:
+        self.ok("list")
+        self.match("/newton/prep PrepData\n"
+                   "/newton/raw RawData\n"
+                   "/newton/zzz/rawnotch RawNotchedData\n")
+
+        # Notice how they're not empty
+        self.ok("list --detail")
+        eq_(self.captured.count('\n'), 11)
+
+        # Delete some
+        self.ok("destroy /newton/prep")
+        self.ok("list")
+        self.match("/newton/raw RawData\n"
+                   "/newton/zzz/rawnotch RawNotchedData\n")
+
+        self.ok("destroy /newton/zzz/rawnotch")
+        self.ok("list")
+        self.match("/newton/raw RawData\n")
+
+        self.ok("destroy /newton/raw")
+        self.ok("create /newton/raw RawData")
+        self.ok("destroy /newton/raw")
+        self.ok("list")
+        self.match("")
+
+        # Re-create a previously deleted location, and some new ones
+        rebuild = [ "/newton/prep", "/newton/zzz",
+                    "/newton/raw", "/newton/asdf/qwer" ]
+        for path in rebuild:
+            # Create the path
+            self.ok("create " + path + " PrepData")
+            self.ok("list")
+            self.contain(path)
+            # Make sure it was created empty
+            self.ok("list --detail --path " + path)
+            self.contain("(no intervals)")

tests/test_interval.py

@@ -13,13 +13,21 @@ from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
 from test_helpers import *
 import unittest
 
+# set to False to skip live renders
+do_live_renders = False
+def render(iset, description = "", live = True):
+    import renderdot
+    r = renderdot.RBTreeRenderer(iset.tree)
+    return r.render(description, live and do_live_renders)
+
 def makeset(string):
     """Build an IntervalSet from a string, for testing purposes
 
     Each character is 1 second
     [ = interval start
-    | = interval end + adjacent start
+    | = interval end + next start
     ] = interval end
+    . = zero-width interval (identical start and end)
     anything else is ignored
     """
     iset = IntervalSet()
@@ -30,9 +38,11 @@ def makeset(string):
         elif (c == "|"):
             iset += Interval(start, day)
             start = day
-        elif (c == "]"):
+        elif (c == ")"):
             iset += Interval(start, day)
             del start
+        elif (c == "."):
+            iset += Interval(day, day)
     return iset
 
 class TestInterval:
@@ -68,24 +78,24 @@ class TestInterval:
         assert(Interval(d1, d3) < Interval(d2, d3))
         assert(Interval(d2, d2) > Interval(d1, d3))
         assert(Interval(d3, d3) == Interval(d3, d3))
-        with assert_raises(AttributeError):
-            x = (i == 123)
+        #with assert_raises(TypeError): # was AttributeError, that's wrong
+        #    x = (i == 123)
 
         # subset
-        assert(Interval(d1, d3).subset(d1, d2) == Interval(d1, d2))
+        eq_(Interval(d1, d3).subset(d1, d2), Interval(d1, d2))
         with assert_raises(IntervalError):
             x = Interval(d2, d3).subset(d1, d2)
 
         # big integers and floats
         x = Interval(5000111222, 6000111222)
-        eq_(str(x), "[5000111222.0 -> 6000111222.0]")
+        eq_(str(x), "[5000111222.0 -> 6000111222.0)")
         x = Interval(123.45, 234.56)
-        eq_(str(x), "[123.45 -> 234.56]")
+        eq_(str(x), "[123.45 -> 234.56)")
 
         # misc
         i = Interval(d1, d2)
         eq_(repr(i), repr(eval(repr(i))))
-        eq_(str(i), "[1332561600.0 -> 1332648000.0]")
+        eq_(str(i), "[1332561600.0 -> 1332648000.0)")
 
     def test_interval_intersect(self):
         # Test Interval intersections
@@ -106,7 +116,7 @@ class TestInterval:
             except IntervalError:
                 assert(i not in should_intersect[True] and
                        i not in should_intersect[False])
-        with assert_raises(AttributeError):
+        with assert_raises(TypeError):
             x = i1.intersects(1234)
 
     def test_intervalset_construct(self):
@@ -165,54 +175,81 @@ class TestInterval:
 
         # misc
         eq_(repr(iset), repr(eval(repr(iset))))
-        eq_(str(iset), "[[100.0 -> 200.0], [200.0 -> 300.0]]")
+        eq_(str(iset), "[[100.0 -> 200.0), [200.0 -> 300.0)]")
 
     def test_intervalset_geniset(self):
         # Test basic iset construction
-        assert(makeset("  [----]   ") ==
-               makeset("  [-|--]   "))
+        eq_(makeset("  [----)   "),
+            makeset("  [-|--)   "))
 
-        assert(makeset("[]  [--]   ") +
-               makeset(" []    [--]") ==
-               makeset("[|] [-----]"))
+        eq_(makeset("[)  [--)   ") +
+            makeset(" [)    [--)"),
+            makeset("[|) [-----)"))
 
-        assert(makeset("  [-------]") ==
-               makeset("  [-|-----|"))
+        eq_(makeset("  [-------)"),
+            makeset("  [-|-----|"))
 
 
     def test_intervalset_intersect(self):
         # Test intersection (&)
-        with assert_raises(AttributeError):
-            x = makeset("[--]") & 1234
+        with assert_raises(TypeError): # was AttributeError
+            x = makeset("[--)") & 1234
 
-        assert(makeset("[---------]") &
-               makeset(" [---]     ") ==
-               makeset(" [---]     "))
+        # Intersection with interval
+        eq_(makeset("[---|---)[)") &
+            list(makeset("  [------) "))[0],
+            makeset("  [-----)  "))
 
-        assert(makeset(" [---]     ") &
-               makeset("[---------]") ==
-               makeset(" [---]     "))
+        # Intersection with sets
+        eq_(makeset("[---------)") &
+            makeset(" [---)     "),
+            makeset(" [---)     "))
 
-        assert(makeset("    [-----]") &
-               makeset(" [-----]   ") ==
-               makeset("    [--]   "))
+        eq_(makeset(" [---)     ") &
+            makeset("[---------)"),
+            makeset(" [---)     "))
 
-        assert(makeset("      [---]") &
-               makeset(" [--]      ") ==
-               makeset("           "))
+        eq_(makeset("    [-----)") &
+            makeset(" [-----)   "),
+            makeset("    [--)   "))
 
-        assert(makeset("    [-|---]") &
-               makeset(" [-----|-] ") ==
-               makeset("    [----] "))
+        eq_(makeset(" [--)  [--)") &
+            makeset("  [------) "),
+            makeset("  [-)  [-) "))
 
-        assert(makeset("    [-|-]  ") &
-               makeset(" [-|--|--] ") ==
-               makeset("    [---]  "))
+        eq_(makeset("      [---)") &
+            makeset(" [--)      "),
+            makeset("           "))
 
-        assert(makeset(" [----][--]") &
-               makeset("[-] [--] []") ==
-               makeset(" [] [-]  []"))
+        eq_(makeset("    [-|---)") &
+            makeset(" [-----|-) "),
+            makeset("    [----) "))
 
+        eq_(makeset("    [-|-)  ") &
+            makeset(" [-|--|--) "),
+            makeset("    [---)  "))
+
+        # Border cases -- will give different results if intervals are
+        # half open or fully closed.  Right now, they are half open,
+        # although that's a little messy since the database intervals
+        # often contain a data point at the endpoint.
+        half_open = True
+        if half_open:
+            eq_(makeset("      [---)") &
+                makeset(" [----)    "),
+                makeset("           "))
+            eq_(makeset(" [----)[--)") &
+                makeset("[-) [--) [)"),
+                makeset(" [) [-)  [)"))
+        else:
+            eq_(makeset("      [---)") &
+                makeset(" [----)    "),
+                makeset("      .    "))
+            eq_(makeset(" [----)[--)") &
+                makeset("[-) [--) [)"),
+                makeset(" [) [-). [)"))
+
+class TestIntervalDB:
     def test_dbinterval(self):
         # Test DBInterval class
         i = DBInterval(100, 200, 100, 200, 10000, 20000)
@@ -255,25 +292,64 @@ class TestInterval:
         for i in IntervalSet(iseta.intersection(Interval(125,250))):
             assert(isinstance(i, DBInterval))
 
+class TestIntervalTree:
+
+    def test_interval_tree(self):
+        import random
+        random.seed(1234)
+
+        # make a set of 100 intervals
+        iset = IntervalSet()
+        j = 100
+        for i in random.sample(xrange(j),j):
+            interval = Interval(i, i+1)
+            iset += interval
+        render(iset, "Random Insertion")
+
+        # remove about half of them
+        for i in random.sample(xrange(j),j):
+            if random.randint(0,1):
+                iset -= Interval(i, i+1)
+
+        # try removing an interval that doesn't exist
+        with assert_raises(IntervalError):
+            iset -= Interval(1234,5678)
+        render(iset, "Random Insertion, deletion")
+
+        # make a set of 100 intervals, inserted in order
+        iset = IntervalSet()
+        j = 100
+        for i in xrange(j):
+            interval = Interval(i, i+1)
+            iset += interval
+        render(iset, "In-order insertion")
+
 class TestIntervalSpeed:
-    #@unittest.skip("this is slow")
+    @unittest.skip("this is slow")
     def test_interval_speed(self):
         import yappi
         import time
         import aplotter
+        import random
+        import math
 
         print
         yappi.start()
         speeds = {}
-        for j in [ 2**x for x in range(5,22) ]:
+        for j in [ 2**x for x in range(5,20) ]:
             start = time.time()
             iset = IntervalSet()
-            for i in xrange(j):
+            for i in random.sample(xrange(j),j):
                 interval = Interval(i, i+1)
                 iset += interval
             speed = (time.time() - start) * 1000000.0
-            printf("%d: %g μs (%g μs each)\n", j, speed, speed/j)
+            printf("%d: %g μs (%g μs each, O(n log n) ratio %g)\n",
+                   j,
+                   speed,
+                   speed/j,
+                   speed / (j*math.log(j))) # should be constant
             speeds[j] = speed
         aplotter.plot(speeds.keys(), speeds.values(), plot_slope=True)
         yappi.stop()
         yappi.print_stats(sort_type=yappi.SORTTYPE_TTOT, limit=10)
+

tests/test_rbtree.py

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+
+import nilmdb
+from nilmdb.printf import *
+
+from nose.tools import *
+from nose.tools import assert_raises
+
+from nilmdb.rbtree import RBTree, RBNode
+
+from test_helpers import *
+import unittest
+
+# set to False to skip live renders
+do_live_renders = False
+def render(tree, description = "", live = True):
+    import renderdot
+    r = renderdot.RBTreeRenderer(tree)
+    return r.render(description, live and do_live_renders)
+
+class TestRBTree:
+    def test_rbtree(self):
+        rb = RBTree()
+        rb.insert(RBNode(10000, 10001))
+        rb.insert(RBNode(10004, 10007))
+        rb.insert(RBNode(10001, 10002))
+        # There was a typo that gave the RBTree a loop in this case.
+        # Verify that the dot isn't too big.
+        s = render(rb, live = False)
+        assert(len(s.splitlines()) < 30)
+
+    def test_rbtree_big(self):
+        import random
+        random.seed(1234)
+
+        # make a set of 100 intervals, inserted in order
+        rb = RBTree()
+        j = 100
+        for i in xrange(j):
+            rb.insert(RBNode(i, i+1))
+        render(rb, "in-order insert")
+
+        # remove about half of them
+        for i in random.sample(xrange(j),j):
+            if random.randint(0,1):
+                rb.delete(rb.find(i, i+1))
+        render(rb, "in-order insert, random delete")
+
+        # make a set of 100 intervals, inserted at random
+        rb = RBTree()
+        j = 100
+        for i in random.sample(xrange(j),j):
+            rb.insert(RBNode(i, i+1))
+        render(rb, "random insert")
+
+        # remove about half of them
+        for i in random.sample(xrange(j),j):
+            if random.randint(0,1):
+                rb.delete(rb.find(i, i+1))
+        render(rb, "random insert, random delete")
+
+        # in-order insert of 50 more
+        for i in xrange(50):
+            rb.insert(RBNode(i+500, i+501))
+        render(rb, "random insert, random delete, in-order insert")
+
+    def test_rbtree_basics(self):
+        rb = RBTree()
+        vals = [ 7, 14, 1, 2, 8, 11, 5, 15, 4]
+        for n in vals:
+            rb.insert(RBNode(n, n))
+
+        # stringify
+        s = ""
+        for node in rb:
+            s += str(node)
+        in_("[node (None) 1", s)
+        eq_(str(rb.nil), "[node nil]")
+
+        # inorder traversal, successor and predecessor
+        last = 0
+        for node in rb:
+            assert(node.start > last)
+            last = node.start
+            successor = rb.successor(node)
+            if successor:
+                assert(rb.predecessor(successor) is node)
+            predecessor = rb.predecessor(node)
+            if predecessor:
+                assert(rb.successor(predecessor) is node)
+
+        # Delete node not in the tree
+        with assert_raises(AttributeError):
+            rb.delete(RBNode(1,2))
+
+        # Delete all nodes!
+        for node in rb:
+            rb.delete(node)
+
+        # Build it up again, make sure it matches
+        for n in vals:
+            rb.insert(RBNode(n, n))
+        s2 = ""
+        for node in rb:
+            s2 += str(node)
+        assert(s == s2)
+
+    def test_rbtree_find(self):
+        # Get a little bit of coverage for some overlapping cases,
+        # even though the class doesn't fully support it.
+        rb = RBTree()
+        nodes = [ RBNode(1, 5), RBNode(1, 10), RBNode(1, 15) ]
+        for n in nodes:
+            rb.insert(n)
+        assert(rb.find(1, 5) is nodes[0])
+        assert(rb.find(1, 10) is nodes[1])
+        assert(rb.find(1, 15) is nodes[2])
+
+    def test_rbtree_find_leftright(self):
+        # Now let's get some ranges in there
+        rb = RBTree()
+        vals = [ 7, 14, 1, 2, 8, 11, 5, 15, 4]
+        for n in vals:
+            rb.insert(RBNode(n*10, n*10+5))
+
+        # Check find_end_left, find_right_start
+        for i in range(160):
+            left = rb.find_left_end(i)
+            right = rb.find_right_start(i)
+            if left:
+                # endpoint should be more than i
+                assert(left.end >= i)
+                # all earlier nodes should have a lower endpoint
+                for node in rb:
+                    if node is left:
+                        break
+                    assert(node.end < i)
+            if right:
+                # startpoint should be less than i
+                assert(right.start <= i)
+                # all later nodes should have a higher startpoint
+                for node in reversed(list(rb)):
+                    if node is right:
+                        break
+                    assert(node.start > i)
+
+    def test_rbtree_intersect(self):
+        # Fill with some ranges
+        rb = RBTree()
+        rb.insert(RBNode(10,20))
+        rb.insert(RBNode(20,25))
+        rb.insert(RBNode(30,40))
+        # Just a quick test; test_interval will do better.
+        eq_(len(list(rb.intersect(1,100))), 3)
+        eq_(len(list(rb.intersect(10,20))), 1)
+        eq_(len(list(rb.intersect(5,15))), 1)
+        eq_(len(list(rb.intersect(15,15))), 1)
+        eq_(len(list(rb.intersect(20,21))), 1)
+        eq_(len(list(rb.intersect(19,21))), 2)

time-bxintersect

@@ -0,0 +1,54 @@
+nosetests
+
+32: 386 μs (12.0625 μs each)
+64: 672.102 μs (10.5016 μs each)
+128: 1510.86 μs (11.8036 μs each)
+256: 2782.11 μs (10.8676 μs each)
+512: 5591.87 μs (10.9216 μs each)
+1024: 12812.1 μs (12.5119 μs each)
+2048: 21835.1 μs (10.6617 μs each)
+4096: 46059.1 μs (11.2449 μs each)
+8192: 114127 μs (13.9315 μs each)
+16384: 181217 μs (11.0606 μs each)
+32768: 419649 μs (12.8067 μs each)
+65536: 804320 μs (12.2729 μs each)
+131072: 1.73534e+06 μs (13.2396 μs each)
+262144: 3.74451e+06 μs (14.2842 μs each)
+524288: 8.8694e+06 μs (16.917 μs each)
+1048576: 1.69993e+07 μs (16.2118 μs each)
+2097152: 3.29387e+07 μs (15.7064 μs each)
+   |                                                                            
+   +3.29387e+07                                                             *   
+   |                                                                    ----    
+   |                                                               -----        
+   |                                                           ----             
+   |                                                      -----                 
+   |                                                 -----                      
+   |                                             ----                           
+   |                                        -----                               
+   |                                   -----                                    
+   |                               ----                                         
+   |                          -----                                             
+   |                      ----                                                  
+   |                 -----                                                      
+   |              ---                                                           
+   |           ---                                                              
+   |        ---                                                                 
+   | -------                                                                    
+---+386---------------------------------------------------------------------+---
+   +32                                                          +2.09715e+06    
+
+name                                 #n            tsub      ttot      tavg
+..vl/lees/bucket/nilm/nilmdb/nilmdb/interval.py.__iadd__:184  4194272          10.025323     30.262723     0.000007    
+..evl/lees/bucket/nilm/nilmdb/nilmdb/interval.py.__init__:27  4194272          24.715377     24.715377     0.000006    
+../lees/bucket/nilm/nilmdb/nilmdb/interval.py.intersects:239  4194272          6.705053      12.577620     0.000003    
+..im/devl/lees/bucket/nilm/nilmdb/tests/aplotter.py.plot:404  1                0.000048      0.001412      0.001412    
+../lees/bucket/nilm/nilmdb/tests/aplotter.py.plot_double:311  1                0.000106      0.001346      0.001346    
+..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.plot_data:201  1                0.000098      0.000672      0.000672    
+..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.plot_line:241  16               0.000298      0.000496      0.000031    
+..jim/devl/lees/bucket/nilm/nilmdb/nilmdb/printf.py.printf:4  17               0.000252      0.000334      0.000020    
+..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.transposed:39  1                0.000229      0.000235      0.000235    
+..vl/lees/bucket/nilm/nilmdb/tests/aplotter.py.y_reversed:45  1                0.000151      0.000174      0.000174    
+
+name           tid              fname                      ttot      scnt
+_MainThread           47269783682784        ..b/python2.7/threading.py.setprofile:88  64.746000     1              

timeit.sh

@@ -1,8 +1,9 @@
+./nilmtool.py destroy /bpnilm/2/raw
 ./nilmtool.py create /bpnilm/2/raw RawData
 
 if true; then
-    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/2/raw
-    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/2/raw
+    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 -r 8000 /bpnilm/2/raw
+    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 -r 8000 /bpnilm/2/raw
 else
     for i in $(seq 2000 2050); do
 	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-010001 /bpnilm/2/raw