Tune sqlite to use write-ahead-logging

Enable the following pragmas: synchronous=NORMAL, journal_mode=WAL.
This offers a significant speedup to INSERT times compared to
synchronous=FULL, and is roughly the same as synchronous=OFF
but should be a bit safer.

nilmdb/scripts/nilmdb_server.py

@@ -25,9 +25,6 @@ def main():
                        default = os.path.join(os.getcwd(), "db"))
     group.add_argument('-q', '--quiet', help = 'Silence output',
                        action = 'store_true')
-    group.add_argument('-n', '--nosync', help = 'Use asynchronous '
-                       'commits for sqlite transactions',
-                       action = 'store_true', default = False)
     group.add_argument('-t', '--traceback',
                        help = 'Provide tracebacks in client errors',
                        action = 'store_true', default = False)
@@ -41,8 +38,7 @@ def main():
 
     # Create database object.  Needs to be serialized before passing
     # to the Server.
-    db = nilmdb.utils.serializer_proxy(
-        nilmdb.server.NilmDB)(args.database, sync = not args.nosync)
+    db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database)
 
     # Configure the server
     if args.quiet:

nilmdb/server/nilmdb.py

@@ -31,11 +31,9 @@ import bisect
 #    after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
 # 3: at the end of an explicit transaction, e.g. "with self.con as con:"
 #
-# To speed up testing, or if this transaction speed becomes an issue,
-# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF.
-
-
-# Don't touch old entries -- just add new ones.
+# To speed things up, we can set 'PRAGMA synchronous=OFF'.  Or, it
+# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL'
+# give an equivalent speedup more safely.  That is what is used here.
 _sql_schema_updates = {
     0: """
     -- All streams
@@ -77,7 +75,7 @@ _sql_schema_updates = {
 class NilmDB(object):
     verbose = 0
 
-    def __init__(self, basepath, sync=True, max_results=None,
+    def __init__(self, basepath, max_results=None,
                  bulkdata_args=None):
         if bulkdata_args is None:
             bulkdata_args = {}
@@ -101,10 +99,8 @@ class NilmDB(object):
         self._sql_schema_update()
 
         # See big comment at top about the performance implications of this
-        if sync:
-            self.con.execute("PRAGMA synchronous=FULL")
-        else:
-            self.con.execute("PRAGMA synchronous=OFF")
+        self.con.execute("PRAGMA synchronous=NORMAL")
+        self.con.execute("PRAGMA journal_mode=WAL")
 
         # Approximate largest number of elements that we want to send
         # in a single reply (for stream_intervals, stream_extract)

tests/test_client.py

@@ -35,8 +35,7 @@ def setup_module():
     recursive_unlink(testdb)
 
     # Start web app on a custom port
-    test_db = nilmdb.utils.serializer_proxy(
-        nilmdb.server.NilmDB)(testdb, sync = False)
+    test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(testdb)
     test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",
                                        port = 32180, stoppable = False,
                                        fast_shutdown = True,

tests/test_cmdline.py

@@ -24,7 +24,7 @@ def server_start(max_results = None, bulkdata_args = {}):
     global test_server, test_db
     # Start web app on a custom port
     test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(
-        testdb, sync = False,
+        testdb,
         max_results = max_results,
         bulkdata_args = bulkdata_args)
     test_server = nilmdb.server.Server(test_db, host = "127.0.0.1",

tests/test_nilmdb.py

@@ -33,7 +33,7 @@ class Test00Nilmdb(object):  # named 00 so it runs first
 
         db = nilmdb.server.NilmDB(testdb)
         db.close()
-        db = nilmdb.server.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         db.close()
 
         # test timer, just to get coverage
@@ -46,7 +46,7 @@ class Test00Nilmdb(object):  # named 00 so it runs first
         in_("test: ", capture.getvalue())
 
     def test_stream(self):
-        db = nilmdb.server.NilmDB(testdb, sync=False)
+        db = nilmdb.server.NilmDB(testdb)
         eq_(db.stream_list(), [])
 
         # Bad path
@@ -104,7 +104,7 @@ class Test00Nilmdb(object):  # named 00 so it runs first
 
 class TestBlockingServer(object):
     def setUp(self):
-        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb, sync=False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
 
     def tearDown(self):
         self.db.close()
@@ -145,7 +145,7 @@ class TestServer(object):
 
     def setUp(self):
         # Start web app on a custom port
-        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb, sync=False)
+        self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb)
         self.server = nilmdb.server.Server(self.db, host = "127.0.0.1",
                                            port = 32180, stoppable = False)
         self.server.start(blocking = False)