add .gitignore

.gitignore

@@ -1,4 +1,2 @@
-db/
-tests/*testdb/
 .coverage
 *.pyc

README.txt

@@ -1,4 +1,2 @@
 sudo apt-get install python-nose python-coverage
-sudo apt-get install python-tables python-cherrypy3
-sudo apt-get install cython # 0.17.1-1 or newer
-
+sudo apt-get install python-tables cython python-cherrypy3

TODO

@@ -1 +1,5 @@
 - Merge adjacent intervals on insert (maybe with client help?)
+
+- Better testing:
+  - see about getting coverage on layout.pyx
+  - layout.pyx performance tests, before and after generalization

design.md

@@ -103,13 +103,13 @@ Speed
 
 - First approach was quadratic.  Adding four hours of data:
 
-    $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/1/raw
+    $ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/1/raw 
 	real    24m31.093s
-	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/1/raw
+	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/1/raw 
 	real    43m44.528s
-	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-130002 /bpnilm/1/raw
+	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-130002 /bpnilm/1/raw 
 	real    93m29.713s
-	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-140003 /bpnilm/1/raw
+	$ time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-140003 /bpnilm/1/raw 
 	real    166m53.007s
 
 - Disabling pytables indexing didn't help:
@@ -122,19 +122,19 @@ Speed
 - Server RAM usage is constant.
 
 - Speed problems were due to IntervalSet speed, of parsing intervals
-  from the database and adding the new one each time.
+  from the database and adding the new one each time. 
 
   - First optimization is to cache result of `nilmdb:_get_intervals`,
     which gives the best speedup.
-
+	
   - Also switched to internally using bxInterval from bx-python package.
     Speed of `tests/test_interval:TestIntervalSpeed` is pretty decent
 	and seems to be growing logarithmically now.  About 85μs per insertion
 	for inserting 131k entries.
-
+	
   - Storing the interval data in SQL might be better, with a scheme like:
     http://www.logarithmic.net/pfh/blog/01235197474
-
+  
 - Next slowdown target is nilmdb.layout.Parser.parse().
   - Rewrote parsers using cython and sscanf
   - Stats (rev 10831), with _add_interval disabled
@@ -142,7 +142,7 @@ Speed
 	 layout.pyx.parse:63               13913 sec, 5.1g calls
 	 numpy:records.py.fromrecords:569   7410 sec, 262k calls
   - Probably OK for now.
-
+    
 IntervalSet speed
 -----------------
 - Initial implementation was pretty slow, even with binary search in
@@ -161,18 +161,6 @@ IntervalSet speed
 - Might be algorithmic improvements to be made in Interval.py,
   like in `__and__`
 
-- Replaced again with rbtree.  Seems decent.  Numbers are time per
-  insert for 2**17 insertions, followed by total wall time and RAM
-  usage for running "make test" with `test_rbtree` and `test_interval`
-  with range(5,20):
-  - old values with bxinterval:
-    20.2 μS, total 20 s, 177 MB RAM
-  - rbtree, plain python:
-    97 μS, total 105 s, 846 MB RAM
-  - rbtree converted to cython:
-    26 μS, total 29 s, 320 MB RAM
-  - rbtree and interval converted to cython:
-    8.4 μS, total 12 s, 134 MB RAM
 
 Layouts
 -------
@@ -182,12 +170,12 @@ just collections and counts of a single type.  We'll still use strings
 to describe them, with format:
 
     type_count
-
+  
 where type is "uint16", "float32", or "float64", and count is an integer.
 
 nilmdb.layout.named() will parse these strings into the appropriate
 handlers.  For compatibility:
-
+  
     "RawData" == "uint16_6"
     "RawNotchedData" == "uint16_9"
     "PrepData" == "float32_8"

nilmdb/RedBlackTree.cc

@@ -1,605 +0,0 @@
-
-
-
-// The RedBlackEntry class is an Abstract Base Class.  This means that no
-// instance of the RedBlackEntry class can exist.  Only classes which
-// inherit from the RedBlackEntry class can exist.  Furthermore any class
-// which inherits from the RedBlackEntry class must define the member
-// function GetKey().  The Print() member function does not have to
-// be defined because a default definition exists.
-//
-// The GetKey() function should return an integer key for that entry.
-// The key for an entry should never change otherwise bad things might occur.
-
-class RedBlackEntry {
-public:
-  RedBlackEntry();
-  virtual ~RedBlackEntry();
-  virtual int GetKey() const = 0;
-  virtual void Print() const;
-};
-
-class RedBlackTreeNode {
-  friend class RedBlackTree;
-public:
-  void Print(RedBlackTreeNode*,
-	     RedBlackTreeNode*) const;
-  RedBlackTreeNode();
-  RedBlackTreeNode(RedBlackEntry *);
-  RedBlackEntry * GetEntry() const;
-  ~RedBlackTreeNode();
-protected:
-  RedBlackEntry * storedEntry;
-  int key;
-  int red; /* if red=0 then the node is black */
-  RedBlackTreeNode * left;
-  RedBlackTreeNode * right;
-  RedBlackTreeNode * parent;
-};
-
-
-
-class RedBlackTree {
-public:
-  RedBlackTree();
-  ~RedBlackTree();
-  void Print() const;
-  RedBlackEntry * DeleteNode(RedBlackTreeNode *);
-  RedBlackTreeNode * Insert(RedBlackEntry *);
-  RedBlackTreeNode * GetPredecessorOf(RedBlackTreeNode *) const;
-  RedBlackTreeNode * GetSuccessorOf(RedBlackTreeNode *) const;
-  RedBlackTreeNode * Search(int key);
-  TemplateStack<RedBlackTreeNode *> * Enumerate(int low, int high) ;
-  void CheckAssumptions() const;
-protected:
-  /*  A sentinel is used for root and for nil.  These sentinels are */
-  /*  created when RedBlackTreeCreate is caled.  root->left should always */
-  /*  point to the node which is the root of the tree.  nil points to a */
-  /*  node which should always be black but has aribtrary children and */
-  /*  parent and no key or info.  The point of using these sentinels is so */
-  /*  that the root and nil nodes do not require special cases in the code */
-  RedBlackTreeNode * root;
-  RedBlackTreeNode * nil;
-  void LeftRotate(RedBlackTreeNode *);
-  void RightRotate(RedBlackTreeNode *);
-  void TreeInsertHelp(RedBlackTreeNode *);
-  void TreePrintHelper(RedBlackTreeNode *) const;
-  void FixUpMaxHigh(RedBlackTreeNode *);
-  void DeleteFixUp(RedBlackTreeNode *);
-};
-
-const int MIN_INT=-MAX_INT;
-
-RedBlackTreeNode::RedBlackTreeNode(){
-};
-
-RedBlackTreeNode::RedBlackTreeNode(RedBlackEntry * newEntry)
-  : storedEntry (newEntry) , key(newEntry->GetKey()) {
-};
-
-RedBlackTreeNode::~RedBlackTreeNode(){
-};
-
-RedBlackEntry * RedBlackTreeNode::GetEntry() const {return storedEntry;}
-
-RedBlackEntry::RedBlackEntry(){
-};
-RedBlackEntry::~RedBlackEntry(){
-};
-void RedBlackEntry::Print() const {
-  cout << "No Print Method defined. Using Default: " << GetKey() << endl;
-}
-
-RedBlackTree::RedBlackTree()
-{
-  nil = new RedBlackTreeNode;
-  nil->left = nil->right = nil->parent = nil;
-  nil->red = 0;
-  nil->key = MIN_INT;
-  nil->storedEntry = NULL;
-
-  root = new RedBlackTreeNode;
-  root->parent = root->left = root->right = nil;
-  root->key = MAX_INT;
-  root->red=0;
-  root->storedEntry = NULL;
-}
-
-/***********************************************************************/
-/*  FUNCTION:  LeftRotate */
-/**/
-/*  INPUTS:  the node to rotate on */
-/**/
-/*  OUTPUT:  None */
-/**/
-/*  Modifies Input: this, x */
-/**/
-/*  EFFECTS:  Rotates as described in _Introduction_To_Algorithms by */
-/*            Cormen, Leiserson, Rivest (Chapter 14).  Basically this */
-/*            makes the parent of x be to the left of x, x the parent of */
-/*            its parent before the rotation and fixes other pointers */
-/*            accordingly.  */
-/***********************************************************************/
-
-void RedBlackTree::LeftRotate(RedBlackTreeNode* x) {
-  RedBlackTreeNode* y;
-
-  /*  I originally wrote this function to use the sentinel for */
-  /*  nil to avoid checking for nil.  However this introduces a */
-  /*  very subtle bug because sometimes this function modifies */
-  /*  the parent pointer of nil.  This can be a problem if a */
-  /*  function which calls LeftRotate also uses the nil sentinel */
-  /*  and expects the nil sentinel's parent pointer to be unchanged */
-  /*  after calling this function.  For example, when DeleteFixUP */
-  /*  calls LeftRotate it expects the parent pointer of nil to be */
-  /*  unchanged. */
-
-  y=x->right;
-  x->right=y->left;
-
-  if (y->left != nil) y->left->parent=x; /* used to use sentinel here */
-  /* and do an unconditional assignment instead of testing for nil */
-
-  y->parent=x->parent;
-
-  /* instead of checking if x->parent is the root as in the book, we */
-  /* count on the root sentinel to implicitly take care of this case */
-  if( x == x->parent->left) {
-    x->parent->left=y;
-  } else {
-    x->parent->right=y;
-  }
-  y->left=x;
-  x->parent=y;
-}
-
-/***********************************************************************/
-/*  FUNCTION:  RighttRotate */
-/**/
-/*  INPUTS:  node to rotate on */
-/**/
-/*  OUTPUT:  None */
-/**/
-/*  Modifies Input?: this, y */
-/**/
-/*  EFFECTS:  Rotates as described in _Introduction_To_Algorithms by */
-/*            Cormen, Leiserson, Rivest (Chapter 14).  Basically this */
-/*            makes the parent of x be to the left of x, x the parent of */
-/*            its parent before the rotation and fixes other pointers */
-/*            accordingly.  */
-/***********************************************************************/
-
-void RedBlackTree::RightRotate(RedBlackTreeNode* y) {
-  RedBlackTreeNode* x;
-
-  /*  I originally wrote this function to use the sentinel for */
-  /*  nil to avoid checking for nil.  However this introduces a */
-  /*  very subtle bug because sometimes this function modifies */
-  /*  the parent pointer of nil.  This can be a problem if a */
-  /*  function which calls LeftRotate also uses the nil sentinel */
-  /*  and expects the nil sentinel's parent pointer to be unchanged */
-  /*  after calling this function.  For example, when DeleteFixUP */
-  /*  calls LeftRotate it expects the parent pointer of nil to be */
-  /*  unchanged. */
-
-  x=y->left;
-  y->left=x->right;
-
-  if (nil != x->right)  x->right->parent=y; /*used to use sentinel here */
-  /* and do an unconditional assignment instead of testing for nil */
-
-  /* instead of checking if x->parent is the root as in the book, we */
-  /* count on the root sentinel to implicitly take care of this case */
-  x->parent=y->parent;
-  if( y == y->parent->left) {
-    y->parent->left=x;
-  } else {
-    y->parent->right=x;
-  }
-  x->right=y;
-  y->parent=x;
-}
-
-/***********************************************************************/
-/*  FUNCTION:  TreeInsertHelp  */
-/**/
-/*  INPUTS:  z is the node to insert */
-/**/
-/*  OUTPUT:  none */
-/**/
-/*  Modifies Input:  this, z */
-/**/
-/*  EFFECTS:  Inserts z into the tree as if it were a regular binary tree */
-/*            using the algorithm described in _Introduction_To_Algorithms_ */
-/*            by Cormen et al.  This funciton is only intended to be called */
-/*            by the Insert function and not by the user */
-/***********************************************************************/
-
-void RedBlackTree::TreeInsertHelp(RedBlackTreeNode* z) {
-  /*  This function should only be called by RedBlackTree::Insert */
-  RedBlackTreeNode* x;
-  RedBlackTreeNode* y;
-
-  z->left=z->right=nil;
-  y=root;
-  x=root->left;
-  while( x != nil) {
-    y=x;
-    if ( x->key > z->key) {
-      x=x->left;
-    } else { /* x->key <= z->key */
-      x=x->right;
-    }
-  }
-  z->parent=y;
-  if ( (y == root) ||
-       (y->key > z->key) ) {
-    y->left=z;
-  } else {
-    y->right=z;
-  }
-}
-
-/*  Before calling InsertNode  the node x should have its key set */
-
-/***********************************************************************/
-/*  FUNCTION:  InsertNode */
-/**/
-/*  INPUTS:  newEntry is the entry to insert*/
-/**/
-/*  OUTPUT:  This function returns a pointer to the newly inserted node */
-/*           which is guarunteed to be valid until this node is deleted. */
-/*           What this means is if another data structure stores this */
-/*           pointer then the tree does not need to be searched when this */
-/*           is to be deleted. */
-/**/
-/*  Modifies Input: tree */
-/**/
-/*  EFFECTS:  Creates a node node which contains the appropriate key and */
-/*            info pointers and inserts it into the tree. */
-/***********************************************************************/
-/* jim */
-RedBlackTreeNode * RedBlackTree::Insert(RedBlackEntry * newEntry)
-{
-  RedBlackTreeNode * y;
-  RedBlackTreeNode * x;
-  RedBlackTreeNode * newNode;
-
-  x = new RedBlackTreeNode(newEntry);
-  TreeInsertHelp(x);
-  newNode = x;
-  x->red=1;
-  while(x->parent->red) { /* use sentinel instead of checking for root */
-    if (x->parent == x->parent->parent->left) {
-      y=x->parent->parent->right;
-      if (y->red) {
-	x->parent->red=0;
-	y->red=0;
-	x->parent->parent->red=1;
-	x=x->parent->parent;
-      } else {
-	if (x == x->parent->right) {
-	  x=x->parent;
-	  LeftRotate(x);
-	}
-	x->parent->red=0;
-	x->parent->parent->red=1;
-	RightRotate(x->parent->parent);
-      }
-    } else { /* case for x->parent == x->parent->parent->right */
-             /* this part is just like the section above with */
-             /* left and right interchanged */
-      y=x->parent->parent->left;
-      if (y->red) {
-	x->parent->red=0;
-	y->red=0;
-	x->parent->parent->red=1;
-	x=x->parent->parent;
-      } else {
-	if (x == x->parent->left) {
-	  x=x->parent;
-	  RightRotate(x);
-	}
-	x->parent->red=0;
-	x->parent->parent->red=1;
-	LeftRotate(x->parent->parent);
-      }
-    }
-  }
-  root->left->red=0;
-  return(newNode);
-}
-
-/***********************************************************************/
-/*  FUNCTION:  GetSuccessorOf  */
-/**/
-/*    INPUTS:  x is the node we want the succesor of */
-/**/
-/*    OUTPUT:  This function returns the successor of x or NULL if no */
-/*             successor exists. */
-/**/
-/*    Modifies Input: none */
-/**/
-/*    Note:  uses the algorithm in _Introduction_To_Algorithms_ */
-/***********************************************************************/
-
-RedBlackTreeNode * RedBlackTree::GetSuccessorOf(RedBlackTreeNode * x) const
-{
-  RedBlackTreeNode* y;
-
-  if (nil != (y = x->right)) { /* assignment to y is intentional */
-    while(y->left != nil) { /* returns the minium of the right subtree of x */
-      y=y->left;
-    }
-    return(y);
-  } else {
-    y=x->parent;
-    while(x == y->right) { /* sentinel used instead of checking for nil */
-      x=y;
-      y=y->parent;
-    }
-    if (y == root) return(nil);
-    return(y);
-  }
-}
-
-/***********************************************************************/
-/*  FUNCTION:  GetPredecessorOf  */
-/**/
-/*    INPUTS:  x is the node to get predecessor of */
-/**/
-/*    OUTPUT:  This function returns the predecessor of x or NULL if no */
-/*             predecessor exists. */
-/**/
-/*    Modifies Input: none */
-/**/
-/*    Note:  uses the algorithm in _Introduction_To_Algorithms_ */
-/***********************************************************************/
-
-RedBlackTreeNode * RedBlackTree::GetPredecessorOf(RedBlackTreeNode * x) const {
-  RedBlackTreeNode* y;
-
-  if (nil != (y = x->left)) { /* assignment to y is intentional */
-    while(y->right != nil) { /* returns the maximum of the left subtree of x */
-      y=y->right;
-    }
-    return(y);
-  } else {
-    y=x->parent;
-    while(x == y->left) {
-      if (y == root) return(nil);
-      x=y;
-      y=y->parent;
-    }
-    return(y);
-  }
-}
-
-/***********************************************************************/
-/*  FUNCTION:  Print */
-/**/
-/*    INPUTS:  none */
-/**/
-/*    OUTPUT:  none  */
-/**/
-/*    EFFECTS:  This function recursively prints the nodes of the tree */
-/*              inorder. */
-/**/
-/*    Modifies Input: none */
-/**/
-/*    Note:    This function should only be called from ITTreePrint */
-/***********************************************************************/
-
-void RedBlackTreeNode::Print(RedBlackTreeNode * nil,
-			     RedBlackTreeNode * root) const {
-  storedEntry->Print();
-  printf(", key=%i ",key);
-  printf("  l->key=");
-  if( left == nil) printf("NULL"); else printf("%i",left->key);
-  printf("  r->key=");
-  if( right == nil) printf("NULL"); else printf("%i",right->key);
-  printf("  p->key=");
-  if( parent == root) printf("NULL"); else printf("%i",parent->key);
-  printf("  red=%i\n",red);
-}
-
-void RedBlackTree::TreePrintHelper( RedBlackTreeNode* x) const {
-
-  if (x != nil) {
-    TreePrintHelper(x->left);
-    x->Print(nil,root);
-    TreePrintHelper(x->right);
-  }
-}
-
-/***********************************************************************/
-/*  FUNCTION:  Print */
-/**/
-/*    INPUTS:  none */
-/**/
-/*    OUTPUT:  none */
-/**/
-/*    EFFECT:  This function recursively prints the nodes of the tree */
-/*             inorder. */
-/**/
-/*    Modifies Input: none */
-/**/
-/***********************************************************************/
-
-void RedBlackTree::Print() const {
-  TreePrintHelper(root->left);
-}
-
-/***********************************************************************/
-/*  FUNCTION:  DeleteFixUp */
-/**/
-/*    INPUTS:  x is the child of the spliced */
-/*             out node in DeleteNode. */
-/**/
-/*    OUTPUT:  none */
-/**/
-/*    EFFECT:  Performs rotations and changes colors to restore red-black */
-/*             properties after a node is deleted */
-/**/
-/*    Modifies Input: this, x */
-/**/
-/*    The algorithm from this function is from _Introduction_To_Algorithms_ */
-/***********************************************************************/
-
-void RedBlackTree::DeleteFixUp(RedBlackTreeNode* x) {
-  RedBlackTreeNode * w;
-  RedBlackTreeNode * rootLeft = root->left;
-
-  while( (!x->red) && (rootLeft != x)) {
-    if (x == x->parent->left) {
-
-//
-      w=x->parent->right;
-      if (w->red) {
-	w->red=0;
-	x->parent->red=1;
-	LeftRotate(x->parent);
-	w=x->parent->right;
-      }
-      if ( (!w->right->red) && (!w->left->red) ) {
-	w->red=1;
-	x=x->parent;
-      } else {
-	if (!w->right->red) {
-	  w->left->red=0;
-	  w->red=1;
-	  RightRotate(w);
-	  w=x->parent->right;
-	}
-	w->red=x->parent->red;
-	x->parent->red=0;
-	w->right->red=0;
-	LeftRotate(x->parent);
-	x=rootLeft; /* this is to exit while loop */
-      }
-//
-
-    } else { /* the code below is has left and right switched from above */
-      w=x->parent->left;
-      if (w->red) {
-	w->red=0;
-	x->parent->red=1;
-	RightRotate(x->parent);
-	w=x->parent->left;
-      }
-      if ( (!w->right->red) && (!w->left->red) ) {
-	w->red=1;
-	x=x->parent;
-      } else {
-	if (!w->left->red) {
-	  w->right->red=0;
-	  w->red=1;
-	  LeftRotate(w);
-	  w=x->parent->left;
-	}
-	w->red=x->parent->red;
-	x->parent->red=0;
-	w->left->red=0;
-	RightRotate(x->parent);
-	x=rootLeft; /* this is to exit while loop */
-      }
-    }
-  }
-  x->red=0;
-
-}
-
-
-/***********************************************************************/
-/*  FUNCTION:  DeleteNode */
-/**/
-/*    INPUTS:  tree is the tree to delete node z from */
-/**/
-/*    OUTPUT:  returns the RedBlackEntry stored at deleted node */
-/**/
-/*    EFFECT:  Deletes z from tree and but don't call destructor */
-/**/
-/*    Modifies Input:  z */
-/**/
-/*    The algorithm from this function is from _Introduction_To_Algorithms_ */
-/***********************************************************************/
-
-RedBlackEntry * RedBlackTree::DeleteNode(RedBlackTreeNode * z){
-  RedBlackTreeNode* y;
-  RedBlackTreeNode* x;
-  RedBlackEntry * returnValue = z->storedEntry;
-
-  y= ((z->left == nil) || (z->right == nil)) ? z : GetSuccessorOf(z);
-  x= (y->left == nil) ? y->right : y->left;
-  if (root == (x->parent = y->parent)) { /* assignment of y->p to x->p is intentional */
-    root->left=x;
-  } else {
-    if (y == y->parent->left) {
-      y->parent->left=x;
-    } else {
-      y->parent->right=x;
-    }
-  }
-  if (y != z) { /* y should not be nil in this case */
-
-    /* y is the node to splice out and x is its child */
-
-    y->left=z->left;
-    y->right=z->right;
-    y->parent=z->parent;
-    z->left->parent=z->right->parent=y;
-    if (z == z->parent->left) {
-      z->parent->left=y;
-    } else {
-      z->parent->right=y;
-    }
-    if (!(y->red)) {
-      y->red = z->red;
-      DeleteFixUp(x);
-    } else
-      y->red = z->red;
-    delete z;
-  } else {
-    if (!(y->red)) DeleteFixUp(x);
-    delete y;
-  }
-  return returnValue;
-}
-
-
-/***********************************************************************/
-/*  FUNCTION:  Enumerate */
-/**/
-/*    INPUTS:  tree is the tree to look for keys between [low,high] */
-/**/
-/*    OUTPUT:  stack containing pointers to the nodes between [low,high] */
-/**/
-/*    Modifies Input: none */
-/**/
-/*    EFFECT:  Returns a stack containing pointers to nodes containing */
-/*             keys which in [low,high]/ */
-/**/
-/***********************************************************************/
-
-TemplateStack<RedBlackTreeNode *> * RedBlackTree::Enumerate(int low,
-							    int high)  {
-  TemplateStack<RedBlackTreeNode *> * enumResultStack =
-    new TemplateStack<RedBlackTreeNode *>(4);
-
-  RedBlackTreeNode* x=root->left;
-  RedBlackTreeNode* lastBest=NULL;
-
-  while(nil != x) {
-    if ( x->key > high ) {
-      x=x->left;
-    } else {
-      lastBest=x;
-      x=x->right;
-    }
-  }
-  while ( (lastBest) && (low <= lastBest->key) ) {
-    enumResultStack->Push(lastBest);
-    lastBest=GetPredecessorOf(lastBest);
-  }
-  return(enumResultStack);
-}

nilmdb/bxintersect.pyx

@@ -1,26 +1,7 @@
 # cython: profile=False
-# This is from bx-python 554:07aca5a9f6fc (BSD licensed), modified to
-# store interval ranges as doubles rather than 32-bit integers.
-
-"""
-Data structure for performing intersect queries on a set of intervals which
-preserves all information about the intervals (unlike bitset projection methods).
-
-:Authors: James Taylor (james@jamestaylor.org),
-          Ian Schenk (ian.schenck@gmail.com),
-          Brent Pedersen (bpederse@gmail.com)
-"""
-
-# Historical note:
-#    This module original contained an implementation based on sorted endpoints
-#    and a binary search, using an idea from Scott Schwartz and Piotr Berman.
-#    Later an interval tree implementation was implemented by Ian for Galaxy's
-#    join tool (see `bx.intervals.operations.quicksect.py`). This was then
-#    converted to Cython by Brent, who also added support for
-#    upstream/downstream/neighbor queries. This was modified by James to
-#    handle half-open intervals strictly, to maintain sort order, and to
-#    implement the same interface as the original Intersecter.
-
+# This is based on bxintersect in bx-python 554:07aca5a9f6fc (BSD licensed);
+# modified to store interval ranges as doubles rather than 32-bit integers,
+# use fully closed intervals, support deletion, etc.
 #cython: cdivision=True
 
 import operator
@@ -194,76 +175,6 @@ cdef class IntervalNode:
             self.cright._intersect( start, end, results )
 
 
-    cdef void _seek_left(IntervalNode self, double position, list results, int n, double max_dist):
-        # we know we can bail in these 2 cases.
-        if self.maxend + max_dist < position:
-            return
-        if self.minstart > position:
-            return
-
-        # the ordering of these 3 blocks makes it so the results are
-        # ordered nearest to farest from the query position
-        if self.cright is not EmptyNode:
-            self.cright._seek_left(position, results, n, max_dist)
-
-        if -1 < position - self.end < max_dist:
-            results.append(self.interval)
-
-        # TODO: can these conditionals be more stringent?
-        if self.cleft is not EmptyNode:
-                self.cleft._seek_left(position, results, n, max_dist)
-
-
-
-    cdef void _seek_right(IntervalNode self, double position, list results, int n, double max_dist):
-        # we know we can bail in these 2 cases.
-        if self.maxend < position: return
-        if self.minstart - max_dist > position: return
-
-        #print "SEEK_RIGHT:",self, self.cleft, self.maxend, self.minstart, position
-
-        # the ordering of these 3 blocks makes it so the results are
-        # ordered nearest to farest from the query position
-        if self.cleft is not EmptyNode:
-                self.cleft._seek_right(position, results, n, max_dist)
-
-        if -1 < self.start - position < max_dist:
-            results.append(self.interval)
-
-        if self.cright is not EmptyNode:
-                self.cright._seek_right(position, results, n, max_dist)
-
-
-    cpdef left(self, position, int n=1, double max_dist=2500):
-        """
-        find n features with a start > than `position`
-        f: a Interval object (or anything with an `end` attribute)
-        n: the number of features to return
-        max_dist: the maximum distance to look before giving up.
-        """
-        cdef list results = []
-        # use start - 1 becuase .left() assumes strictly left-of
-        self._seek_left( position - 1, results, n, max_dist )
-        if len(results) == n: return results
-        r = results
-        r.sort(key=operator.attrgetter('end'), reverse=True)
-        return r[:n]
-
-    cpdef right(self, position, int n=1, double max_dist=2500):
-        """
-        find n features with a end < than position
-        f: a Interval object (or anything with a `start` attribute)
-        n: the number of features to return
-        max_dist: the maximum distance to look before giving up.
-        """
-        cdef list results = []
-        # use end + 1 becuase .right() assumes strictly right-of
-        self._seek_right(position + 1, results, n, max_dist)
-        if len(results) == n: return results
-        r = results
-        r.sort(key=operator.attrgetter('start'))
-        return r[:n]
-
     def traverse(self):
         if self.cleft is not EmptyNode:
             for node in self.cleft.traverse():
@@ -392,6 +303,7 @@ cdef class IntervalTree:
 
     # ---- Position based interfaces -----------------------------------------
 
+    ## KEEP
     def insert( self, double start, double end, object value=None ):
         """
         Insert the interval [start,end) associated with value `value`.
@@ -401,8 +313,14 @@ cdef class IntervalTree:
         else:
             self.root = self.root.insert( start, end, value )
 
-    add = insert
-
+    def delete( self, double start, double end, object value=None ):
+        """
+        Delete the interval [start,end) associated with value `value`.
+        """
+        if self.root is None:
+            self.root = IntervalNode( start, end, value )
+        else:
+            self.root = self.root.insert( start, end, value )
 
     def find( self, start, end ):
         """
@@ -412,26 +330,9 @@ cdef class IntervalTree:
             return []
         return self.root.find( start, end )
 
-    def before( self, position, num_intervals=1, max_dist=2500 ):
-        """
-        Find `num_intervals` intervals that lie before `position` and are no
-        further than `max_dist` positions away
-        """
-        if self.root is None:
-            return []
-        return self.root.left( position, num_intervals, max_dist )
-
-    def after( self, position, num_intervals=1, max_dist=2500 ):
-        """
-        Find `num_intervals` intervals that lie after `position` and are no
-        further than `max_dist` positions away
-        """
-        if self.root is None:
-            return []
-        return self.root.right( position, num_intervals, max_dist )
-
     # ---- Interval-like object based interfaces -----------------------------
 
+    ## KEEP
     def insert_interval( self, interval ):
         """
         Insert an "interval" like object (one with at least start and end
@@ -439,50 +340,6 @@ cdef class IntervalTree:
         """
         self.insert( interval.start, interval.end, interval )
 
-    add_interval = insert_interval
-
-    def before_interval( self, interval, num_intervals=1, max_dist=2500 ):
-        """
-        Find `num_intervals` intervals that lie completely before `interval`
-        and are no further than `max_dist` positions away
-        """
-        if self.root is None:
-            return []
-        return self.root.left( interval.start, num_intervals, max_dist )
-
-    def after_interval( self, interval, num_intervals=1, max_dist=2500 ):
-        """
-        Find `num_intervals` intervals that lie completely after `interval` and
-        are no further than `max_dist` positions away
-        """
-        if self.root is None:
-            return []
-        return self.root.right( interval.end, num_intervals, max_dist )
-
-    def upstream_of_interval( self, interval, num_intervals=1, max_dist=2500 ):
-        """
-        Find `num_intervals` intervals that lie completely upstream of
-        `interval` and are no further than `max_dist` positions away
-        """
-        if self.root is None:
-            return []
-        if interval.strand == -1 or interval.strand == "-":
-            return self.root.right( interval.end, num_intervals, max_dist )
-        else:
-            return self.root.left( interval.start, num_intervals, max_dist )
-
-    def downstream_of_interval( self, interval, num_intervals=1, max_dist=2500 ):
-        """
-        Find `num_intervals` intervals that lie completely downstream of
-        `interval` and are no further than `max_dist` positions away
-        """
-        if self.root is None:
-            return []
-        if interval.strand == -1 or interval.strand == "-":
-            return self.root.left( interval.start, num_intervals, max_dist )
-        else:
-            return self.root.right( interval.end, num_intervals, max_dist )
-
     def traverse(self):
         """
         iterator that traverses the tree

nilmdb/client.py

@@ -84,11 +84,6 @@ class Client(object):
                    "layout" : layout }
         return self.http.get("stream/create", params)
 
-    def stream_destroy(self, path):
-        """Delete stream and its contents"""
-        params = { "path": path }
-        return self.http.get("stream/destroy", params)
-
     def stream_insert(self, path, data):
         """Insert data into a stream.  data should be a file-like object
         that provides ASCII data that matches the database layout for path."""

nilmdb/cmdline/cmdline.py

@@ -15,8 +15,7 @@ version = "0.1"
 
 # Valid subcommands.  Defined in separate files just to break
 # things up -- they're still called with Cmdline as self.
-subcommands = [ "info", "create", "list", "metadata", "insert", "extract",
-                "destroy" ]
+subcommands = [ "info", "create", "list", "metadata", "insert", "extract" ]
 
 # Import the subcommand modules.  Equivalent way of doing this would be
 # from . import info as cmd_info

nilmdb/cmdline/destroy.py

@@ -1,25 +0,0 @@
-from __future__ import absolute_import
-from nilmdb.printf import *
-import nilmdb.client
-
-from argparse import ArgumentDefaultsHelpFormatter as def_form
-
-def setup(self, sub):
-    cmd = sub.add_parser("destroy", help="Delete a stream and all data",
-                         formatter_class = def_form,
-                         description="""
-                         Destroy the stream at the specified path.  All
-                         data and metadata related to the stream is
-                         permanently deleted.
-                         """)
-    cmd.set_defaults(handler = cmd_destroy)
-    group = cmd.add_argument_group("Required arguments")
-    group.add_argument("path",
-                       help="Path of the stream to delete, e.g. /foo/bar")
-
-def cmd_destroy(self):
-    """Destroy stream"""
-    try:
-        self.client.stream_destroy(self.args.path)
-    except nilmdb.client.ClientError as e:
-        self.die("Error deleting stream: %s", str(e))

nilmdb/interval.py

@@ -1,9 +1,8 @@
-"""Interval, IntervalSet
+"""Interval and IntervalSet
 
 Represents an interval of time, and a set of such intervals.
 
-Intervals are half-open, ie. they include data points with timestamps
-[start, end)
+Intervals are closed, ie. they include timestamps [start, end]
 """
 
 # First implementation kept a sorted list of intervals and used
@@ -19,20 +18,18 @@ Intervals are half-open, ie. they include data points with timestamps
 # Fourth version is an optimized rb-tree that stores interval starts
 # and ends directly in the tree, like bxinterval did.
 
-cimport rbtree
-cdef extern from "stdint.h":
-    ctypedef unsigned long long uint64_t
+# Fifth version is back to modified bxintersect...
+
+import bxintersect
 
 class IntervalError(Exception):
     """Error due to interval overlap, etc"""
     pass
 
-cdef class Interval:
+class Interval(object):
     """Represents an interval of time."""
 
-    cdef public double start, end
-
-    def __init__(self, double start, double end):
+    def __init__(self, start, end):
         """
         'start' and 'end' are arbitrary floats that represent time
         """
@@ -46,9 +43,9 @@ cdef class Interval:
         return self.__class__.__name__ + "(" + s + ")"
 
     def __str__(self):
-        return "[" + repr(self.start) + " -> " + repr(self.end) + ")"
+        return "[" + str(self.start) + " -> " + str(self.end) + "]"
 
-    def __cmp__(self, Interval other):
+    def __cmp__(self, other):
         """Compare two intervals.  If non-equal, order by start then end"""
         if not isinstance(other, Interval):
             raise TypeError("bad type")
@@ -62,20 +59,20 @@ cdef class Interval:
             return -1
         return 1
 
-    cpdef intersects(self, Interval other):
+    def intersects(self, other):
         """Return True if two Interval objects intersect"""
         if (self.end <= other.start or self.start >= other.end):
             return False
         return True
 
-    cpdef subset(self, double start, double end):
+    def subset(self, start, end):
         """Return a new Interval that is a subset of this one"""
         # A subclass that tracks additional data might override this.
         if start < self.start or end > self.end:
             raise IntervalError("not a subset")
         return Interval(start, end)
 
-cdef class DBInterval(Interval):
+class DBInterval(Interval):
     """
     Like Interval, but also tracks corresponding start/end times and
     positions within the database.  These are not currently modified
@@ -90,9 +87,6 @@ cdef class DBInterval(Interval):
         db_end = 200, db_endpos = 20000
     """
 
-    cpdef public double db_start, db_end
-    cpdef public uint64_t db_startpos, db_endpos
-
     def __init__(self, start, end,
                  db_start, db_end,
                  db_startpos, db_endpos):
@@ -117,7 +111,7 @@ cdef class DBInterval(Interval):
         s += ", " + repr(self.db_startpos) + ", " + repr(self.db_endpos)
         return self.__class__.__name__ + "(" + s + ")"
 
-    cpdef subset(self, double start, double end):
+    def subset(self, start, end):
         """
         Return a new DBInterval that is a subset of this one
         """
@@ -127,18 +121,16 @@ cdef class DBInterval(Interval):
                           self.db_start, self.db_end,
                           self.db_startpos, self.db_endpos)
 
-cdef class IntervalSet:
+class IntervalSet(object):
     """
     A non-intersecting set of intervals.
     """
 
-    cdef public rbtree.RBTree tree
-
     def __init__(self, source=None):
         """
         'source' is an Interval or IntervalSet to add.
         """
-        self.tree = rbtree.RBTree()
+        self.tree = bxinterval.IntervalTree()
         if source is not None:
             self += source
 
@@ -158,7 +150,7 @@ cdef class IntervalSet:
         descs = [ str(x) for x in self ]
         return  "[" + ", ".join(descs) + "]"
 
-    def __match__(self, other):
+    def __eq__(self, other):
         # This isn't particularly efficient, but it shouldn't get used in the
         # general case.
         """Test equality of two IntervalSets.
@@ -209,20 +201,10 @@ cdef class IntervalSet:
         except IndexError:
             return False
 
-    # Use __richcmp__ instead of __eq__, __ne__ for Cython.
-    def __richcmp__(self, other, int op):
-        if op == 2: # ==
-            return self.__match__(other)
-        elif op == 3: # !=
-            return not self.__match__(other)
-        return False
-    #def __eq__(self, other):
-    #    return self.__match__(other)
-    #
-    #def __ne__(self, other):
-    #    return not self.__match__(other)
+    def __ne__(self, other):
+        return not self.__eq__(other)
 
-    def __iadd__(self, object other not None):
+    def __iadd__(self, other):
         """Inplace add -- modifies self
 
         This throws an exception if the regions being added intersect."""
@@ -230,13 +212,13 @@ cdef class IntervalSet:
             if self.intersects(other):
                 raise IntervalError("Tried to add overlapping interval "
                                     "to this set")
-            self.tree.insert(rbtree.RBNode(other.start, other.end, other))
+            self.tree.insert_interval(other)
         else:
             for x in other:
                 self.__iadd__(x)
         return self
 
-    def __isub__(self, Interval other not None):
+    def __isub__(self, other):
         """Inplace subtract -- modifies self
 
         Removes an interval from the set.  Must exist exactly
@@ -247,13 +229,13 @@ cdef class IntervalSet:
         self.tree.delete(i)
         return self
 
-    def __add__(self, other not None):
+    def __add__(self, other):
         """Add -- returns a new object"""
         new = IntervalSet(self)
         new += IntervalSet(other)
         return new
 
-    def __and__(self, other not None):
+    def __and__(self, other):
         """
         Compute a new IntervalSet from the intersection of two others
 
@@ -264,15 +246,15 @@ cdef class IntervalSet:
 
         if not isinstance(other, IntervalSet):
             for i in self.intersection(other):
-                out.tree.insert(rbtree.RBNode(i.start, i.end, i))
+                out.tree.insert(rbtree.RBNode(i))
         else:
             for x in other:
                 for i in self.intersection(x):
-                    out.tree.insert(rbtree.RBNode(i.start, i.end, i))
+                    out.tree.insert(rbtree.RBNode(i))
 
         return out
 
-    def intersection(self, Interval interval not None):
+    def intersection(self, interval):
         """
         Compute a sequence of intervals that correspond to the
         intersection between `self` and the provided interval.
@@ -289,14 +271,23 @@ cdef class IntervalSet:
             if i:
                 if i.start >= interval.start and i.end <= interval.end:
                     yield i
+                elif i.start > interval.end:
+                    break
                 else:
                     subset = i.subset(max(i.start, interval.start),
                                       min(i.end, interval.end))
                     yield subset
 
-    cpdef intersects(self, Interval other):
+    def intersects(self, other):
+        ### PROBABLY WRONG
         """Return True if this IntervalSet intersects another interval"""
-        for n in self.tree.intersect(other.start, other.end):
-            if n.obj.intersects(other):
-                return True
+        node = self.tree.find_left(other.start, other.end)
+        if node is None:
+            return False
+        for n in self.tree.inorder(node):
+            if n.obj:
+                if n.obj.intersects(other):
+                    return True
+                if n.obj > other:
+                    break
         return False

nilmdb/interval.pyxdep

@@ -1 +0,0 @@
-rbtree.pxd

nilmdb/nilmdb.py

@@ -291,6 +291,16 @@ class NilmDB(object):
         if group == '':
             raise ValueError("invalid path")
 
+        # Make the group structure, one element at a time
+        group_path = group.lstrip('/').split("/")
+        for i in range(len(group_path)):
+            parent = "/" + "/".join(group_path[0:i])
+            child = group_path[i]
+            try:
+                self.h5file.createGroup(parent, child)
+            except tables.NodeError:
+                pass
+
         # Get description
         try:
             desc = nilmdb.layout.get_named(layout_name).description()
@@ -302,15 +312,9 @@ class NilmDB(object):
         exp_rows = 8000 * 60*60*24*30*3
 
         # Create the table
-        try:
-            table = self.h5file.createTable(group, node,
-                                            description = desc,
-                                            expectedrows = exp_rows,
-                                            createparents = True)
-        except AttributeError:
-            # Trying to create e.g. /foo/bar/baz when /foo/bar is already
-            # a table raises this error.
-            raise ValueError("error creating table at that path")
+        table = self.h5file.createTable(group, node,
+                                        description = desc,
+                                        expectedrows = exp_rows)
 
         # Insert into SQL database once the PyTables is happy
         with self.con as con:
@@ -333,7 +337,8 @@ class NilmDB(object):
            """
         stream_id = self._stream_id(path)
         with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
+            con.execute("DELETE FROM metadata "
+                        "WHERE stream_id=?", (stream_id,))
             for key in data:
                 if data[key] != '':
                     con.execute("INSERT INTO metadata VALUES (?, ?, ?)",
@@ -356,33 +361,6 @@ class NilmDB(object):
         data.update(newdata)
         self.stream_set_metadata(path, data)
 
-    def stream_destroy(self, path):
-        """Fully remove a table and all of its data from the database.
-        No way to undo it!  The group structure is removed, if there
-        are no other tables in it.  Metadata is removed."""
-        stream_id = self._stream_id(path)
-
-        # Delete the cached interval data
-        if stream_id in self._cached_iset:
-            del self._cached_iset[stream_id]
-
-        # Delete the data node, and all parent nodes (if they have no
-        # remaining children)
-        split_path = path.lstrip('/').split("/")
-        while split_path:
-            name = split_path.pop()
-            where = "/" + "/".join(split_path)
-            try:
-                self.h5file.removeNode(where, name, recursive = False)
-            except tables.NodeError:
-                break
-
-        # Delete metadata, stream, intervals
-        with self.con as con:
-            con.execute("DELETE FROM metadata WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM ranges WHERE stream_id=?", (stream_id,))
-            con.execute("DELETE FROM streams WHERE id=?", (stream_id,))
-
     def stream_insert(self, path, parser, old_timestamp = None):
         """Insert new data into the database.
            path: Path at which to add the data
@@ -408,7 +386,7 @@ class NilmDB(object):
         iset = self._get_intervals(stream_id)
         interval = Interval(min_timestamp, parser.max_timestamp)
         if iset.intersects(interval):
-            raise OverlapError("new data overlaps existing data at range: "
+            raise OverlapError("new data overlaps existing data: "
                                + str(iset & interval))
 
         # Insert the data into pytables

nilmdb/rbtree.pxd

@@ -1,23 +0,0 @@
-cdef class RBNode:
-    cdef public object obj
-    cdef public double start, end
-    cdef public int red
-    cdef public RBNode left, right, parent
-
-cdef class RBTree:
-    cdef public RBNode nil, root
-
-    cpdef getroot(RBTree self)
-    cdef void __rotate_left(RBTree self, RBNode x)
-    cdef void __rotate_right(RBTree self, RBNode y)
-    cdef RBNode __successor(RBTree self, RBNode x)
-    cpdef RBNode successor(RBTree self, RBNode x)
-    cdef RBNode __predecessor(RBTree self, RBNode x)
-    cpdef RBNode predecessor(RBTree self, RBNode x)
-    cpdef insert(RBTree self, RBNode z)
-    cdef void __insert_fixup(RBTree self, RBNode x)
-    cpdef delete(RBTree self, RBNode z)
-    cdef inline void __delete_fixup(RBTree self, RBNode x)
-    cpdef RBNode find(RBTree self, double start, double end)
-    cpdef RBNode find_left_end(RBTree self, double t)
-    cpdef RBNode find_right_start(RBTree self, double t)

nilmdb/rbtree.py

@@ -1,27 +1,20 @@
-# cython: profile=False
-# cython: cdivision=True
-
-"""
-Jim Paris <jim@jtan.com>
-
-Red-black tree, where keys are stored as start/end timestamps.
-This is a basic interval tree that holds half-open intervals:
-  [start, end)
-Intervals must not overlap.  Fixing that would involve making this
-into an augmented interval tree as described in CLRS 14.3.
-
-Code that assumes non-overlapping intervals is marked with the
-string 'non-overlapping'.
-"""
+"""Red-black tree, where keys are stored as start/end timestamps."""
 
 import sys
-cimport rbtree
 
-cdef class RBNode:
-    """One node of the Red/Black tree, containing a key (start, end)
-    and value (obj)"""
-    def __init__(self, double start, double end, object obj = None):
+class RBNode(object):
+    """One node of the Red/Black tree.  obj points to any object,
+    'start' and 'end' are timestamps that represent the key."""
+    def __init__(self, obj = None, start = None, end = None):
+        """If given an object but no start/end times, get the
+        start/end times from the object.
+
+        If given start/end times, obj can be anything, including None."""
         self.obj = obj
+        if start is None:
+            start = obj.start
+        if end is None:
+            end = obj.end
         self.start = start
         self.end = end
         self.red = False
@@ -33,23 +26,21 @@ cdef class RBNode:
             color = "R"
         else:
             color = "B"
-        if self.start == sys.float_info.min:
-            return "[node nil]"
-        return ("[node ("
-                + str(self.obj) + ") "
+        return ("[node "
                 + str(self.start) + " -> " + str(self.end) + " "
                 + color + "]")
 
-cdef class RBTree:
+class RBTree(object):
     """Red/Black tree"""
 
     # Init
     def __init__(self):
         self.nil = RBNode(start = sys.float_info.min,
-                          end = sys.float_info.min)
+                           end = sys.float_info.min)
         self.nil.left = self.nil
         self.nil.right = self.nil
         self.nil.parent = self.nil
+        self.nil.nil = True
 
         self.root = RBNode(start = sys.float_info.max,
                            end = sys.float_info.max)
@@ -57,21 +48,9 @@ cdef class RBTree:
         self.root.right = self.nil
         self.root.parent = self.nil
 
-    # We have a dummy root node to simplify operations, so from an
-    # external point of view, its left child is the real root.
-    cpdef getroot(self):
-        return self.root.left
-
     # Rotations and basic operations
-    cdef void __rotate_left(self, RBNode x):
-        """Rotate left:
-        #   x           y
-        #  / \   -->   / \
-        # z   y       x   w
-        #    / \     / \
-        #   v   w   z   v
-        """
-        cdef RBNode y = x.right
+    def __rotate_left(self, x):
+        y = x.right
         x.right = y.left
         if y.left is not self.nil:
             y.left.parent = x
@@ -83,15 +62,8 @@ cdef class RBTree:
         y.left = x
         x.parent = y
 
-    cdef void __rotate_right(self, RBNode y):
-        """Rotate right:
-        #     y           x
-        #    / \   -->   / \
-        #   x   w       z   y
-        #  / \             / \
-        # z   v           v   w
-        """
-        cdef RBNode x = y.left
+    def __rotate_right(self, y):
+        x = y.left
         y.left = x.right
         if x.right is not self.nil:
             x.right.parent = y
@@ -103,9 +75,9 @@ cdef class RBTree:
         x.right = y
         y.parent = x
 
-    cdef RBNode __successor(self, RBNode x):
+    def __successor(self, x):
         """Returns the successor of RBNode x"""
-        cdef RBNode y = x.right
+        y = x.right
         if y is not self.nil:
             while y.left is not self.nil:
                 y = y.left
@@ -117,14 +89,10 @@ cdef class RBTree:
             if y is self.root:
                 return self.nil
         return y
-    cpdef RBNode successor(self, RBNode x):
-        """Returns the successor of RBNode x, or None"""
-        cdef RBNode y = self.__successor(x)
-        return y if y is not self.nil else None
 
-    cdef RBNode __predecessor(self, RBNode x):
+    def _predecessor(self, x):
         """Returns the predecessor of RBNode x"""
-        cdef RBNode y = x.left
+        y = x.left
         if y is not self.nil:
             while y.right is not self.nil:
                 y = y.right
@@ -137,18 +105,14 @@ cdef class RBTree:
                 x = y
                 y = y.parent
         return y
-    cpdef RBNode predecessor(self, RBNode x):
-        """Returns the predecessor of RBNode x, or None"""
-        cdef RBNode y = self.__predecessor(x)
-        return y if y is not self.nil else None
 
     # Insertion
-    cpdef insert(self, RBNode z):
+    def insert(self, z):
         """Insert RBNode z into RBTree and rebalance as necessary"""
         z.left = self.nil
         z.right = self.nil
-        cdef RBNode y = self.root
-        cdef RBNode x = self.root.left
+        y = self.root
+        x = self.root.left
         while x is not self.nil:
             y = x
             if (x.start > z.start or (x.start == z.start and x.end > z.end)):
@@ -164,7 +128,7 @@ cdef class RBTree:
         # relabel/rebalance
         self.__insert_fixup(z)
 
-    cdef void __insert_fixup(self, RBNode x):
+    def __insert_fixup(self, x):
         """Rebalance/fix RBTree after a simple insertion of RBNode x"""
         x.red = True
         while x.parent.red:
@@ -199,11 +163,10 @@ cdef class RBTree:
         self.root.left.red = False
 
     # Deletion
-    cpdef delete(self, RBNode z):
+    def delete(self, z):
         if z.left is None or z.right is None:
             raise AttributeError("you can only delete a node object "
                                  + "from the tree; use find() to get one")
-        cdef RBNode x, y
         if z.left is self.nil or z.right is self.nil:
             y = z
         else:
@@ -240,10 +203,10 @@ cdef class RBTree:
             if not y.red:
                 self.__delete_fixup(x)
 
-    cdef void __delete_fixup(self, RBNode x):
+    def __delete_fixup(self, x):
         """Rebalance/fix RBTree after a deletion.  RBNode x is the
         child of the spliced out node."""
-        cdef RBNode rootLeft = self.root.left
+        rootLeft = self.root.left
         while not x.red and x is not rootLeft:
             if x is x.parent.left:
                 w = x.parent.right
@@ -289,89 +252,141 @@ cdef class RBTree:
                     x = rootLeft # exit loop
         x.red = False
 
+    # Rendering
+    def __render_dot_node(self, node, max_depth = 20):
+        from printf import sprintf
+        """Render a single node and its children into a dot graph fragment"""
+        if max_depth == 0:
+            return ""
+        if node is self.nil:
+            return ""
+        def c(red):
+            if red:
+                return 'color="#ff0000", style=filled, fillcolor="#ffc0c0"'
+            else:
+                return 'color="#000000", style=filled, fillcolor="#c0c0c0"'
+        s = sprintf("%d [label=\"%g\\n%g\", %s];\n",
+                    id(node),
+                    node.start, node.end,
+                    c(node.red))
+
+        if node.left is self.nil:
+            s += sprintf("L%d [label=\"-\", %s];\n", id(node), c(False))
+            s += sprintf("%d -> L%d [label=L];\n", id(node), id(node))
+        else:
+            s += sprintf("%d -> %d [label=L];\n", id(node), id(node.left))
+        if node.right is self.nil:
+            s += sprintf("R%d [label=\"-\", %s];\n", id(node), c(False))
+            s += sprintf("%d -> R%d [label=R];\n", id(node), id(node))
+        else:
+            s += sprintf("%d -> %d [label=R];\n", id(node), id(node.right))
+        s += self.__render_dot_node(node.left, max_depth-1)
+        s += self.__render_dot_node(node.right, max_depth-1)
+        return s
+
+    def render_dot(self, title = "RBTree"):
+        """Render the entire RBTree as a dot graph"""
+        return ("digraph rbtree {\n"
+                + self.__render_dot_node(self.root.left)
+                + "}\n");
+
+    def render_dot_live(self, title = "RBTree"):
+        """Render the entire RBTree as a dot graph, live GTK view"""
+        import gtk
+        import gtk.gdk
+        sys.path.append("/usr/share/xdot")
+        import xdot
+        xdot.Pen.highlighted = lambda pen: pen
+        s = ("digraph rbtree {\n"
+             + self.__render_dot_node(self.root)
+             + "}\n");
+        window = xdot.DotWindow()
+        window.set_dotcode(s)
+        window.set_title(title + " - any key to close")
+        window.connect('destroy', gtk.main_quit)
+        def quit(widget, event):
+            if not event.is_modifier:
+                window.destroy()
+                gtk.main_quit()
+        window.widget.connect('key-press-event', quit)
+        gtk.main()
+
     # Walking, searching
     def __iter__(self):
-        return self.inorder()
+        return self.inorder(self.root.left)
 
-    def inorder(self, RBNode x = None):
+    def inorder(self, x = None):
         """Generator that performs an inorder walk for the tree
-        rooted at RBNode x"""
+        starting at RBNode x"""
         if x is None:
-            x = self.getroot()
+            x = self.root.left
         while x.left is not self.nil:
             x = x.left
         while x is not self.nil:
             yield x
             x = self.__successor(x)
 
-    cpdef RBNode find(self, double start, double end):
-        """Return the node with exactly the given start and end."""
-        cdef RBNode x = self.getroot()
+    def __find_all(self, start, end, x):
+        """Find node with the specified (start,end) key.
+        Also returns the largest node less than or equal to key,
+        and the smallest node greater or equal to than key."""
+        if x is None:
+            x = self.root.left
+        largest = self.nil
+        smallest = self.nil
         while x is not self.nil:
             if start < x.start:
-                x = x.left
+                smallest = x
+                x = x.left # start <
             elif start == x.start:
-                if end == x.end:
-                    break # found it
-                elif end < x.end:
-                    x = x.left
+                if end < x.end:
+                    smallest = x
+                    x = x.left # start =, end <
+                elif end == x.end: # found it
+                    smallest = x
+                    largest = x
+                    break
                 else:
-                    x = x.right
+                    largest = x
+                    x = x.right # start =, end >
             else:
-                x = x.right
-        return x if x is not self.nil else None
+                largest = x
+                x = x.right # start >
+        return (x, smallest, largest)
 
-    cpdef RBNode find_left_end(self, double t):
-        """Find the leftmode node with end >= t.  With non-overlapping
-        intervals, this is the first node that might overlap time t.
+    def find(self, start, end, x = None):
+        """Find node with the key == (start,end), or None"""
+        y = self.__find_all(start, end, x)[1]
+        return y if y is not self.nil else None
 
-        Note that this relies on non-overlapping intervals, since
-        it assumes that we can use the endpoints to traverse the
-        tree even though it was created using the start points."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if t < x.end:
-                if x.left is self.nil:
-                    break
-                x = x.left
-            elif t == x.end:
-                break
-            else:
-                if x.right is self.nil:
-                    x = self.__successor(x)
-                    break
-                x = x.right
-        return x if x is not self.nil else None
+    def find_right(self, start, end, x = None):
+        """Find node with the smallest key >= (start,end), or None"""
+        y = self.__find_all(start, end, x)[1]
+        return y if y is not self.nil else None
 
-    cpdef RBNode find_right_start(self, double t):
-        """Find the rightmode node with start <= t.  With non-overlapping
-        intervals, this is the last node that might overlap time t."""
-        cdef RBNode x = self.getroot()
-        while x is not self.nil:
-            if t < x.start:
-                if x.left is self.nil:
-                    x = self.__predecessor(x)
-                    break
-                x = x.left
-            elif t == x.start:
-                break
-            else:
-                if x.right is self.nil:
-                    break
-                x = x.right
-        return x if x is not self.nil else None
+    def find_left(self, start, end, x = None):
+        """Find node with the largest key <= (start,end), or None"""
+        y = self.__find_all(start, end, x)[2]
+        return y if y is not self.nil else None
 
     # Intersections
-    def intersect(self, double start, double end):
+    def intersect(self, start, end):
         """Generator that returns nodes that overlap the given
-        (start,end) range.  Assumes non-overlapping intervals."""
-        # Start with the leftmode node that ends after start
-        cdef RBNode n = self.find_left_end(start)
-        while n is not None:
-            if n.start >= end:
-                # this node starts after the requested end; we're done
-                break
-            if start < n.end:
-                # this node overlaps our requested area
-                yield n
-            n = self.successor(n)
+        (start,end) range, for the tree rooted at RBNode x.
+
+        NOTE: this assumes non-overlapping intervals."""
+        # Start with the leftmost node before the starting point
+        n = self.find_left(start, start)
+        # If we didn't find one, look for the leftmode node before the
+        # ending point instead.
+        if n is None:
+            n = self.find_left(end, end)
+        # If we still didn't find it, there are no intervals that intersect.
+        if n is None:
+            return none
+
+        # Now yield this node and all successors until their endpoints
+
+        if False:
+            yield
+        return

nilmdb/rbtree.pyxdep

@@ -1 +0,0 @@
-rbtree.pxd

nilmdb/server.py

@@ -88,17 +88,6 @@ class Stream(NilmApp):
             message = sprintf("%s: %s", type(e).__name__, e.message)
             raise cherrypy.HTTPError("400 Bad Request", message)
 
-    # /stream/destroy?path=/newton/prep
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    def destroy(self, path):
-        """Delete a stream and its associated data."""
-        try:
-            return self.db.stream_destroy(path)
-        except Exception as e:
-            message = sprintf("%s: %s", type(e).__name__, e.message)
-            raise cherrypy.HTTPError("400 Bad Request", message)
-
     # /stream/get_metadata?path=/newton/prep
     # /stream/get_metadata?path=/newton/prep&key=foo&key=bar
     @cherrypy.expose

setup.cfg

@@ -13,9 +13,7 @@ verbosity=2
 #tests=tests/test_cmdline.py
 #tests=tests/test_layout.py
 #tests=tests/test_rbtree.py
-#tests=tests/test_interval.py
-#tests=tests/test_rbtree.py,tests/test_interval.py
-#tests=tests/test_interval.py
+tests=tests/test_interval.py
 #tests=tests/test_client.py
 #tests=tests/test_timestamper.py
 #tests=tests/test_serializer.py

tests/renderdot.py

@@ -1,90 +0,0 @@
-import sys
-
-class Renderer(object):
-
-    def __init__(self, getleft, getright,
-                 getred, getstart, getend, nil):
-        self.getleft = getleft
-        self.getright = getright
-        self.getred = getred
-        self.getstart = getstart
-        self.getend = getend
-        self.nil = nil
-
-    # Rendering
-    def __render_dot_node(self, node, max_depth = 20):
-        from nilmdb.printf import sprintf
-        """Render a single node and its children into a dot graph fragment"""
-        if max_depth == 0:
-            return ""
-        if node is self.nil:
-            return ""
-        def c(red):
-            if red:
-                return 'color="#ff0000", style=filled, fillcolor="#ffc0c0"'
-            else:
-                return 'color="#000000", style=filled, fillcolor="#c0c0c0"'
-        s = sprintf("%d [label=\"%g\\n%g\", %s];\n",
-                    id(node),
-                    self.getstart(node), self.getend(node),
-                    c(self.getred(node)))
-
-        if self.getleft(node) is self.nil:
-            s += sprintf("L%d [label=\"-\", %s];\n", id(node), c(False))
-            s += sprintf("%d -> L%d [label=L];\n", id(node), id(node))
-        else:
-            s += sprintf("%d -> %d [label=L];\n",
-                         id(node),id(self.getleft(node)))
-        if self.getright(node) is self.nil:
-            s += sprintf("R%d [label=\"-\", %s];\n", id(node), c(False))
-            s += sprintf("%d -> R%d [label=R];\n", id(node), id(node))
-        else:
-            s += sprintf("%d -> %d [label=R];\n",
-                         id(node), id(self.getright(node)))
-        s += self.__render_dot_node(self.getleft(node), max_depth-1)
-        s += self.__render_dot_node(self.getright(node), max_depth-1)
-        return s
-
-    def render_dot(self, rootnode, title = "Tree"):
-        """Render the entire tree as a dot graph"""
-        return ("digraph rbtree {\n"
-                + self.__render_dot_node(rootnode)
-                + "}\n");
-
-    def render_dot_live(self, rootnode, title = "Tree"):
-        """Render the entiretree as a dot graph, live GTK view"""
-        import gtk
-        import gtk.gdk
-        sys.path.append("/usr/share/xdot")
-        import xdot
-        xdot.Pen.highlighted = lambda pen: pen
-        s = ("digraph rbtree {\n"
-             + self.__render_dot_node(rootnode)
-             + "}\n");
-        window = xdot.DotWindow()
-        window.set_dotcode(s)
-        window.set_title(title + " - any key to close")
-        window.connect('destroy', gtk.main_quit)
-        def quit(widget, event):
-            if not event.is_modifier:
-                window.destroy()
-                gtk.main_quit()
-        window.widget.connect('key-press-event', quit)
-        gtk.main()
-
-class RBTreeRenderer(Renderer):
-    def __init__(self, tree):
-        Renderer.__init__(self,
-                          lambda node: node.left,
-                          lambda node: node.right,
-                          lambda node: node.red,
-                          lambda node: node.start,
-                          lambda node: node.end,
-                          tree.nil)
-        self.tree = tree
-
-    def render(self, title = "RBTree", live = True):
-        if live:
-            return Renderer.render_dot_live(self, self.tree.getroot(), title)
-        else:
-            return Renderer.render_dot(self, self.tree.getroot(), title)

tests/test_cmdline.py

@@ -192,17 +192,11 @@ class TestCmdline(object):
         self.contain("no such layout")
 
         # Create a few streams
-        self.ok("create /newton/zzz/rawnotch RawNotchedData")
         self.ok("create /newton/prep PrepData")
         self.ok("create /newton/raw RawData")
+        self.ok("create /newton/zzz/rawnotch RawNotchedData")
 
-        # Should not be able to create a stream with another stream as
-        # its parent
-        self.fail("create /newton/prep/blah PrepData")
-        self.contain("error creating table at that path")
-
-        # Verify we got those 3 streams and they're returned in
-        # alphabetical order.
+        # Verify we got those 3 streams
         self.ok("list")
         self.match("/newton/prep PrepData\n"
                    "/newton/raw RawData\n"
@@ -462,54 +456,3 @@ class TestCmdline(object):
         eq_(self.captured.count('\n'), 11)
         server_stop()
         server_start()
-
-    def test_cmdline_10_destroy(self):
-        # Delete records
-        self.ok("destroy --help")
-
-        self.fail("destroy")
-        self.contain("too few arguments")
-
-        self.fail("destroy /no/such/stream")
-        self.contain("No stream at path")
-
-        self.fail("destroy asdfasdf")
-        self.contain("No stream at path")
-
-        # From previous tests, we have:
-        self.ok("list")
-        self.match("/newton/prep PrepData\n"
-                   "/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
-
-        # Notice how they're not empty
-        self.ok("list --detail")
-        eq_(self.captured.count('\n'), 11)
-
-        # Delete some
-        self.ok("destroy /newton/prep")
-        self.ok("list")
-        self.match("/newton/raw RawData\n"
-                   "/newton/zzz/rawnotch RawNotchedData\n")
-
-        self.ok("destroy /newton/zzz/rawnotch")
-        self.ok("list")
-        self.match("/newton/raw RawData\n")
-
-        self.ok("destroy /newton/raw")
-        self.ok("create /newton/raw RawData")
-        self.ok("destroy /newton/raw")
-        self.ok("list")
-        self.match("")
-
-        # Re-create a previously deleted location, and some new ones
-        rebuild = [ "/newton/prep", "/newton/zzz",
-                    "/newton/raw", "/newton/asdf/qwer" ]
-        for path in rebuild:
-            # Create the path
-            self.ok("create " + path + " PrepData")
-            self.ok("list")
-            self.contain(path)
-            # Make sure it was created empty
-            self.ok("list --detail --path " + path)
-            self.contain("(no intervals)")

tests/test_interval.py

@@ -13,19 +13,12 @@ from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
 from test_helpers import *
 import unittest
 
-# set to False to skip live renders
-do_live_renders = False
-def render(iset, description = "", live = True):
-    import renderdot
-    r = renderdot.RBTreeRenderer(iset.tree)
-    return r.render(description, live and do_live_renders)
-
 def makeset(string):
     """Build an IntervalSet from a string, for testing purposes
 
     Each character is 1 second
     [ = interval start
-    | = interval end + next start
+    | = interval end + adjacent start
     ] = interval end
     . = zero-width interval (identical start and end)
     anything else is ignored
@@ -38,7 +31,7 @@ def makeset(string):
         elif (c == "|"):
             iset += Interval(start, day)
             start = day
-        elif (c == ")"):
+        elif (c == "]"):
             iset += Interval(start, day)
             del start
         elif (c == "."):
@@ -78,24 +71,24 @@ class TestInterval:
         assert(Interval(d1, d3) < Interval(d2, d3))
         assert(Interval(d2, d2) > Interval(d1, d3))
         assert(Interval(d3, d3) == Interval(d3, d3))
-        #with assert_raises(TypeError): # was AttributeError, that's wrong
-        #    x = (i == 123)
+        with assert_raises(TypeError): # was AttributeError, that's wrong
+            x = (i == 123)
 
         # subset
-        eq_(Interval(d1, d3).subset(d1, d2), Interval(d1, d2))
+        assert(Interval(d1, d3).subset(d1, d2) == Interval(d1, d2))
         with assert_raises(IntervalError):
             x = Interval(d2, d3).subset(d1, d2)
 
         # big integers and floats
         x = Interval(5000111222, 6000111222)
-        eq_(str(x), "[5000111222.0 -> 6000111222.0)")
+        eq_(str(x), "[5000111222.0 -> 6000111222.0]")
         x = Interval(123.45, 234.56)
-        eq_(str(x), "[123.45 -> 234.56)")
+        eq_(str(x), "[123.45 -> 234.56]")
 
         # misc
         i = Interval(d1, d2)
         eq_(repr(i), repr(eval(repr(i))))
-        eq_(str(i), "[1332561600.0 -> 1332648000.0)")
+        eq_(str(i), "[1332561600.0 -> 1332648000.0]")
 
     def test_interval_intersect(self):
         # Test Interval intersections
@@ -116,7 +109,7 @@ class TestInterval:
             except IntervalError:
                 assert(i not in should_intersect[True] and
                        i not in should_intersect[False])
-        with assert_raises(TypeError):
+        with assert_raises(AttributeError):
             x = i1.intersects(1234)
 
     def test_intervalset_construct(self):
@@ -175,79 +168,61 @@ class TestInterval:
 
         # misc
         eq_(repr(iset), repr(eval(repr(iset))))
-        eq_(str(iset), "[[100.0 -> 200.0), [200.0 -> 300.0)]")
+        eq_(str(iset), "[[100.0 -> 200.0], [200.0 -> 300.0]]")
 
     def test_intervalset_geniset(self):
         # Test basic iset construction
-        eq_(makeset("  [----)   "),
-            makeset("  [-|--)   "))
+        assert(makeset("  [----]   ") ==
+               makeset("  [-|--]   "))
 
-        eq_(makeset("[)  [--)   ") +
-            makeset(" [)    [--)"),
-            makeset("[|) [-----)"))
+        assert(makeset("[]  [--]   ") +
+               makeset(" []    [--]") ==
+               makeset("[|] [-----]"))
 
-        eq_(makeset("  [-------)"),
-            makeset("  [-|-----|"))
+        assert(makeset("  [-------]") ==
+               makeset("  [-|-----|"))
 
 
     def test_intervalset_intersect(self):
         # Test intersection (&)
         with assert_raises(TypeError): # was AttributeError
-            x = makeset("[--)") & 1234
+            x = makeset("[--]") & 1234
 
-        # Intersection with interval
-        eq_(makeset("[---|---)[)") &
-            list(makeset("  [------) "))[0],
-            makeset("  [-----)  "))
+        assert(makeset("[---------]") &
+               makeset(" [---]     ") ==
+               makeset(" [---]     "))
 
-        # Intersection with sets
-        eq_(makeset("[---------)") &
-            makeset(" [---)     "),
-            makeset(" [---)     "))
+        assert(makeset(" [---]     ") &
+               makeset("[---------]") ==
+               makeset(" [---]     "))
 
-        eq_(makeset(" [---)     ") &
-            makeset("[---------)"),
-            makeset(" [---)     "))
+        assert(makeset("    [-----]") &
+               makeset(" [-----]   ") ==
+               makeset("    [--]   "))
 
-        eq_(makeset("    [-----)") &
-            makeset(" [-----)   "),
-            makeset("    [--)   "))
+        assert(makeset(" [--]  [--]") &
+               makeset("  [------] ") ==
+               makeset("  [-]  [-] "))
 
-        eq_(makeset(" [--)  [--)") &
-            makeset("  [------) "),
-            makeset("  [-)  [-) "))
+        assert(makeset("      [---]") &
+               makeset(" [--]      ") ==
+               makeset("           "))
 
-        eq_(makeset("      [---)") &
-            makeset(" [--)      "),
-            makeset("           "))
+        assert(makeset("      [---]") &
+               makeset(" [----]    ") ==
+               makeset("      .    "))
 
-        eq_(makeset("    [-|---)") &
-            makeset(" [-----|-) "),
-            makeset("    [----) "))
+        assert(makeset("    [-|---]") &
+               makeset(" [-----|-] ") ==
+               makeset("    [----] "))
 
-        eq_(makeset("    [-|-)  ") &
-            makeset(" [-|--|--) "),
-            makeset("    [---)  "))
+        assert(makeset("    [-|-]  ") &
+               makeset(" [-|--|--] ") ==
+               makeset("    [---]  "))
 
-        # Border cases -- will give different results if intervals are
-        # half open or fully closed.  Right now, they are half open,
-        # although that's a little messy since the database intervals
-        # often contain a data point at the endpoint.
-        half_open = True
-        if half_open:
-            eq_(makeset("      [---)") &
-                makeset(" [----)    "),
-                makeset("           "))
-            eq_(makeset(" [----)[--)") &
-                makeset("[-) [--) [)"),
-                makeset(" [) [-)  [)"))
-        else:
-            eq_(makeset("      [---)") &
-                makeset(" [----)    "),
-                makeset("      .    "))
-            eq_(makeset(" [----)[--)") &
-                makeset("[-) [--) [)"),
-                makeset(" [) [-). [)"))
+        assert(makeset(" [----][--]") &
+               makeset("[-] [--] []") ==
+               makeset(" [] [-]. []"))
 
 class TestIntervalDB:
     def test_dbinterval(self):
@@ -298,13 +273,12 @@ class TestIntervalTree:
         import random
         random.seed(1234)
 
-        # make a set of 100 intervals
+        # make a set of 500 intervals
         iset = IntervalSet()
-        j = 100
+        j = 500
         for i in random.sample(xrange(j),j):
             interval = Interval(i, i+1)
             iset += interval
-        render(iset, "Random Insertion")
 
         # remove about half of them
         for i in random.sample(xrange(j),j):
@@ -314,15 +288,10 @@ class TestIntervalTree:
         # try removing an interval that doesn't exist
         with assert_raises(IntervalError):
             iset -= Interval(1234,5678)
-        render(iset, "Random Insertion, deletion")
 
-        # make a set of 100 intervals, inserted in order
-        iset = IntervalSet()
-        j = 100
-        for i in xrange(j):
-            interval = Interval(i, i+1)
-            iset += interval
-        render(iset, "In-order insertion")
+        # show the graph
+        if False:
+            iset.tree.render_dot_live()
 
 class TestIntervalSpeed:
     @unittest.skip("this is slow")
@@ -331,23 +300,18 @@ class TestIntervalSpeed:
         import time
         import aplotter
         import random
-        import math
 
         print
         yappi.start()
         speeds = {}
-        for j in [ 2**x for x in range(5,20) ]:
+        for j in [ 2**x for x in range(5,18) ]:
             start = time.time()
             iset = IntervalSet()
             for i in random.sample(xrange(j),j):
                 interval = Interval(i, i+1)
                 iset += interval
             speed = (time.time() - start) * 1000000.0
-            printf("%d: %g μs (%g μs each, O(n log n) ratio %g)\n",
-                   j,
-                   speed,
-                   speed/j,
-                   speed / (j*math.log(j))) # should be constant
+            printf("%d: %g μs (%g μs each)\n", j, speed, speed/j)
             speeds[j] = speed
         aplotter.plot(speeds.keys(), speeds.values(), plot_slope=True)
         yappi.stop()

tests/test_rbtree.py

@@ -11,149 +11,65 @@ from nilmdb.rbtree import RBTree, RBNode
 from test_helpers import *
 import unittest
 
-# set to False to skip live renders
-do_live_renders = False
-def render(tree, description = "", live = True):
-    import renderdot
-    r = renderdot.RBTreeRenderer(tree)
-    return r.render(description, live and do_live_renders)
+render = False
 
 class TestRBTree:
     def test_rbtree(self):
         rb = RBTree()
-        rb.insert(RBNode(10000, 10001))
-        rb.insert(RBNode(10004, 10007))
-        rb.insert(RBNode(10001, 10002))
+        rb.insert(RBNode(None, 10000, 10001))
+        rb.insert(RBNode(None, 10004, 10007))
+        rb.insert(RBNode(None, 10001, 10002))
+        s = rb.render_dot()
         # There was a typo that gave the RBTree a loop in this case.
         # Verify that the dot isn't too big.
-        s = render(rb, live = False)
         assert(len(s.splitlines()) < 30)
 
     def test_rbtree_big(self):
         import random
         random.seed(1234)
 
-        # make a set of 100 intervals, inserted in order
+        # make a set of 500 intervals, inserted in order
         rb = RBTree()
-        j = 100
+        j = 500
         for i in xrange(j):
-            rb.insert(RBNode(i, i+1))
-        render(rb, "in-order insert")
+            rb.insert(RBNode(None, i, i+1))
+
+        # show the graph
+        if render:
+            rb.render_dot_live("in-order insert")
 
         # remove about half of them
         for i in random.sample(xrange(j),j):
             if random.randint(0,1):
                 rb.delete(rb.find(i, i+1))
-        render(rb, "in-order insert, random delete")
 
-        # make a set of 100 intervals, inserted at random
+        # show the graph
+        if render:
+            rb.render_dot_live("in-order insert, random delete")
+
+        # make a set of 500 intervals, inserted at random
         rb = RBTree()
-        j = 100
+        j = 500
         for i in random.sample(xrange(j),j):
-            rb.insert(RBNode(i, i+1))
-        render(rb, "random insert")
+            rb.insert(RBNode(None, i, i+1))
+
+        # show the graph
+        if render:
+            rb.render_dot_live("random insert")
 
         # remove about half of them
         for i in random.sample(xrange(j),j):
             if random.randint(0,1):
                 rb.delete(rb.find(i, i+1))
-        render(rb, "random insert, random delete")
 
-        # in-order insert of 50 more
-        for i in xrange(50):
-            rb.insert(RBNode(i+500, i+501))
-        render(rb, "random insert, random delete, in-order insert")
+        # show the graph
+        if render:
+            rb.render_dot_live("random insert, random delete")
 
-    def test_rbtree_basics(self):
-        rb = RBTree()
-        vals = [ 7, 14, 1, 2, 8, 11, 5, 15, 4]
-        for n in vals:
-            rb.insert(RBNode(n, n))
+        # in-order insert of 250 more
+        for i in xrange(250):
+            rb.insert(RBNode(None, i+500, i+501))
 
-        # stringify
-        s = ""
-        for node in rb:
-            s += str(node)
-        in_("[node (None) 1", s)
-        eq_(str(rb.nil), "[node nil]")
-
-        # inorder traversal, successor and predecessor
-        last = 0
-        for node in rb:
-            assert(node.start > last)
-            last = node.start
-            successor = rb.successor(node)
-            if successor:
-                assert(rb.predecessor(successor) is node)
-            predecessor = rb.predecessor(node)
-            if predecessor:
-                assert(rb.successor(predecessor) is node)
-
-        # Delete node not in the tree
-        with assert_raises(AttributeError):
-            rb.delete(RBNode(1,2))
-
-        # Delete all nodes!
-        for node in rb:
-            rb.delete(node)
-
-        # Build it up again, make sure it matches
-        for n in vals:
-            rb.insert(RBNode(n, n))
-        s2 = ""
-        for node in rb:
-            s2 += str(node)
-        assert(s == s2)
-
-    def test_rbtree_find(self):
-        # Get a little bit of coverage for some overlapping cases,
-        # even though the class doesn't fully support it.
-        rb = RBTree()
-        nodes = [ RBNode(1, 5), RBNode(1, 10), RBNode(1, 15) ]
-        for n in nodes:
-            rb.insert(n)
-        assert(rb.find(1, 5) is nodes[0])
-        assert(rb.find(1, 10) is nodes[1])
-        assert(rb.find(1, 15) is nodes[2])
-
-    def test_rbtree_find_leftright(self):
-        # Now let's get some ranges in there
-        rb = RBTree()
-        vals = [ 7, 14, 1, 2, 8, 11, 5, 15, 4]
-        for n in vals:
-            rb.insert(RBNode(n*10, n*10+5))
-
-        # Check find_end_left, find_right_start
-        for i in range(160):
-            left = rb.find_left_end(i)
-            right = rb.find_right_start(i)
-            if left:
-                # endpoint should be more than i
-                assert(left.end >= i)
-                # all earlier nodes should have a lower endpoint
-                for node in rb:
-                    if node is left:
-                        break
-                    assert(node.end < i)
-            if right:
-                # startpoint should be less than i
-                assert(right.start <= i)
-                # all later nodes should have a higher startpoint
-                for node in reversed(list(rb)):
-                    if node is right:
-                        break
-                    assert(node.start > i)
-
-    def test_rbtree_intersect(self):
-        # Fill with some ranges
-        rb = RBTree()
-        rb.insert(RBNode(10,20))
-        rb.insert(RBNode(20,25))
-        rb.insert(RBNode(30,40))
-        # Just a quick test; test_interval will do better.
-        eq_(len(list(rb.intersect(1,100))), 3)
-        eq_(len(list(rb.intersect(10,20))), 1)
-        eq_(len(list(rb.intersect(5,15))), 1)
-        eq_(len(list(rb.intersect(15,15))), 1)
-        eq_(len(list(rb.intersect(20,21))), 1)
-        eq_(len(list(rb.intersect(19,21))), 2)
+        # show the graph
+        if render:
+            rb.render_dot_live("random insert, random delete, in-order insert")

timeit.sh

@@ -1,9 +1,8 @@
-./nilmtool.py destroy /bpnilm/2/raw
 ./nilmtool.py create /bpnilm/2/raw RawData
 
 if true; then
-    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 -r 8000 /bpnilm/2/raw
-    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 -r 8000 /bpnilm/2/raw
+    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-110000 /bpnilm/2/raw
+    time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s 20110513-120001 /bpnilm/2/raw
 else
     for i in $(seq 2000 2050); do
 	time zcat /home/jim/bpnilm-data/snapshot-1-20110513-110002.raw.gz | ./nilmtool.py insert -s ${i}0101-010001 /bpnilm/2/raw