misc

git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10323 ddd99763-3ecb-0310-9145-efcb8ce7c51f

pytables-test/Makefile

@@ -0,0 +1,5 @@
+all:
+	python server.py
+
+clean:
+	rm -f *pyc

pytables-test/README.jim

@@ -0,0 +1,2 @@
+New version from:
+  http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=641485#15

pytables-test/TODO.txt

@@ -0,0 +1,12 @@
+- Make CherryPy server that can handle simple GET/POST,
+  and a httplib client that can talk to that server.
+  Steps:
+  - Make server handle GET
+  - Make client send request, get response
+  - Add request streaming to server
+  - Add request streaming to client
+  - Make server handle POST
+  - Make client send request, get response
+  - Add request streaming to server
+  - Add request streaming to client
+  - Integrate into a server process that also keeps database open.

pytables-test/indexing-notes

@@ -0,0 +1,3 @@
+Indexing time64 doesn't seem to work -- needed to do "time >= 1243052015" even though the actual database times
+should be something like 1243052015.847000.  Let's switch to just using a 64-bit integer counting e.g.
+microseconds since 1970-01-01 

pytables-test/sample-query

@@ -0,0 +1,3 @@
+timestamp > 1243052015
+took 394.5 minutes in vitables
+(2340 rows matched)

pytables-test/server.py

@@ -0,0 +1,20 @@
+import cherrypy
+
+class NilmDB:
+    """NILM Database"""
+
+    @cherrypy.expose
+    def index(self, id = None):
+        return "Base app"
+
+cherrypy.config.update({
+    'server.socket_host': '127.0.0.1',
+    'server.socket_port': 8080
+    })
+
+cherrypy.tree.mount(NilmDB(), "/")
+
+if __name__ == "__main__":
+    cherrypy.server.quickstart()
+    cherrypy.engine.start()
+    cherrypy.engine.block()

pytables-test/speed-pytables.py

@@ -0,0 +1,16 @@
+import tables
+import numpy
+
+class RawSample(tables.IsDescription):
+    timestamp = tables.UInt64Col()
+    voltage   = tables.UInt16Col(shape = 3)
+    current   = tables.UInt16Col(shape = 3)
+
+h5file = tables.openFile("test.h5", mode = "w", title = "Test")
+group = h5file.createGroup("/", "raw", "Raw Data")
+table = h5file.createTable(group, "nilm1", RawSample, "NILM 1")
+
+print repr(h5file)
+
+# write rows
+

pytables-test/test-indexing.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+from tables import *
+import re
+import time
+
+# A class to describe our data
+class PrepData(IsDescription):
+    timestamp = Time64Col()
+    p1 = Float32Col()
+    q1 = Float32Col()
+    p3 = Float32Col()
+    q3 = Float32Col()
+    p5 = Float32Col()
+    q5 = Float32Col()
+    p7 = Float32Col()
+    q7 = Float32Col()
+
+filename = "test.h5"
+h5file = openFile(filename, mode = "w", title = "NILM Test")
+
+group = h5file.createGroup("/", "newton", "Newton school")
+table = h5file.createTable(group, "prep", PrepData, "Prep Data")
+
+table.cols.timestamp.createIndex()
+
+for i in range(0, 80):
+    # Open file
+    data = open("data/alldata")
+    count = 0
+    oldtime = time.time()
+    prep = table.row
+    for line in data:
+        count = count + 1
+        if count % 1000000 == 0:
+            print str(i) + ": " + str((time.time() - oldtime)) + ", total " + str(count/1000000) + "m lines"
+            oldtime = time.time()
+        v = re.split('\s+', line)
+        prep['timestamp'] = float(v[0]) / 1000.0 + 500000 * i
+        prep['p1'] = v[1]
+        prep['q1'] = v[2]
+        prep['p3'] = v[3]
+        prep['q3'] = v[4]
+        prep['p5'] = v[5]
+        prep['q5'] = v[6]
+        prep['p7'] = v[7]
+        prep['q7'] = v[8]
+        prep.append()
+    data.close()
+
+h5file.close()
+
+
+

pytables-test/test-write.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+from tables import *
+import re
+import time
+
+# A class to describe our data
+class PrepData(IsDescription):
+    timestamp = Time64Col()
+    p1 = Float32Col()
+    q1 = Float32Col()
+    p3 = Float32Col()
+    q3 = Float32Col()
+    p5 = Float32Col()
+    q5 = Float32Col()
+    p7 = Float32Col()
+    q7 = Float32Col()
+
+filename = "test.h5"
+h5file = openFile(filename, mode = "w", title = "NILM Test")
+
+group = h5file.createGroup("/", "newton", "Newton school")
+table = h5file.createTable(group, "prep", PrepData, "Prep Data")
+
+table.cols.timestamp.createIndex()
+
+for i in range(0, 80):
+    # Open file
+    data = open("data/alldata")
+    count = 0
+    oldtime = time.time()
+    prep = table.row
+    for line in data:
+        count = count + 1
+        if count % 1000000 == 0:
+            print str(i) + ": " + str((time.time() - oldtime)) + ", total " + str(count/1000000) + "m lines"
+            oldtime = time.time()
+        v = re.split('\s+', line)
+        prep['timestamp'] = float(v[0]) / 1000.0 + 500000 * i
+        prep['p1'] = v[1]
+        prep['q1'] = v[2]
+        prep['p3'] = v[3]
+        prep['q3'] = v[4]
+        prep['p5'] = v[5]
+        prep['q5'] = v[6]
+        prep['p7'] = v[7]
+        prep['q7'] = v[8]
+        prep.append()
+    data.close()
+
+h5file.close()
+
+
+