Fix slow database stuff in testing by setting PRAGMA synchronous=OFF. Default is still FULL for production use, though

git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10639 ddd99763-3ecb-0310-9145-efcb8ce7c51f

nilmdb/nilmdb.py

@@ -20,6 +20,19 @@ import sys
 import os
 import errno
 
+# Note about performance and transactions:
+# 
+# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
+# takes about 125msec.  sqlite3 will commit transactions at 3 times:
+# 1: explicit con.commit()
+# 2: between a series of DML commands and non-DML commands, e.g.
+#    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
+# 3: at the end of an explicit transaction, e.g. "with self.con as con:"
+#
+# To speed up testing, or if this transaction speed becomes an issue,
+# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF.
+
+
 # Don't touch old entries -- just add new ones.
 sql_schema_updates = {
     0: """
@@ -61,7 +74,7 @@ class StreamError(Exception):
 class NilmDB(object):
     verbose = 0
 
-    def __init__(self, basepath):
+    def __init__(self, basepath, sync=True):
         # set up path
         self.basepath = basepath.rstrip('/')
 
@@ -85,6 +98,12 @@ class NilmDB(object):
         self.con = sqlite3.connect(sqlfilename, check_same_thread = False)
         self.sql_schema_update()
 
+        # See big comment at top about the performance implications of this
+        if sync:
+            self.con.execute("PRAGMA synchronous=FULL")
+        else:
+            self.con.execute("PRAGMA synchronous=OFF")
+            
         self.opened = True
 
     def __del__(self):
@@ -112,7 +131,6 @@ class NilmDB(object):
                 printf("Schema updated to %d\n", version)
 
         if version != oldversion:
-            # this takes .1 seconds, so only do it if necessary
             with self.con:
                 cur.execute("PRAGMA user_version = {v:d}".format(v=version))
 

setup.cfg

@@ -10,7 +10,7 @@ cover-erase=1
 stop=1
 verbosity=2
 #tests=tests/test_nilmdb.py
-#tests=tests/test_client.py
+#tests=tests/test_client.py:TestClient.test_client_nilmdb
 #with-profile=1
 #profile-sort=time
-##profile-restrict=10  # doesn't work right, treated as string or something
+##profile-restrict=10  # doesn't work right, treated as string or something

tests/test_client.py

@@ -36,7 +36,7 @@ def setup_module():
         pass
 
     # Start web app on a custom port
-    test_db = nilmdb.NilmDB(testdb)
+    test_db = nilmdb.NilmDB(testdb, sync=False)
     test_server = nilmdb.Server(test_db, host = "127.0.0.1",
                                 port = 12380, stoppable = False,
                                 fast_shutdown = True)

tests/test_nilmdb.py

@@ -36,11 +36,11 @@ class Test00Nilmdb(object):  # named 00 so it runs first
 
         db = nilmdb.NilmDB(testdb)
         db.close()
-        db = nilmdb.NilmDB(testdb)
+        db = nilmdb.NilmDB(testdb, sync=False)
         db.close()
 
     def test_stream(self):
-        db = nilmdb.NilmDB(testdb)
+        db = nilmdb.NilmDB(testdb, sync=False)
         eq_(db.stream_list(), [])
 
         # Bad path
@@ -92,7 +92,7 @@ class Test00Nilmdb(object):  # named 00 so it runs first
 
 class TestBlockingServer(object):
     def setUp(self):
-        self.db = nilmdb.NilmDB(testdb)
+        self.db = nilmdb.NilmDB(testdb, sync=False)
 
     def tearDown(self):
         self.db.close()
@@ -127,7 +127,7 @@ class TestServer(object):
 
     def setUp(self):
         # Start web app on a custom port
-        self.db = nilmdb.NilmDB(testdb)
+        self.db = nilmdb.NilmDB(testdb, sync=False)
         self.server = nilmdb.Server(self.db, host = "127.0.0.1",
                                     port = 12380, stoppable = False)
         self.server.start(blocking = False)