Enable the following pragmas: synchronous=NORMAL, journal_mode=WAL. This offers a significant speedup to INSERT times compared to synchronous=FULL, and is roughly the same as synchronous=OFF but should be a bit safer.tags/nilmdb-1.3.0^0
@@ -25,9 +25,6 @@ def main(): | |||||
default = os.path.join(os.getcwd(), "db")) | default = os.path.join(os.getcwd(), "db")) | ||||
group.add_argument('-q', '--quiet', help = 'Silence output', | group.add_argument('-q', '--quiet', help = 'Silence output', | ||||
action = 'store_true') | action = 'store_true') | ||||
group.add_argument('-n', '--nosync', help = 'Use asynchronous ' | |||||
'commits for sqlite transactions', | |||||
action = 'store_true', default = False) | |||||
group.add_argument('-t', '--traceback', | group.add_argument('-t', '--traceback', | ||||
help = 'Provide tracebacks in client errors', | help = 'Provide tracebacks in client errors', | ||||
action = 'store_true', default = False) | action = 'store_true', default = False) | ||||
@@ -41,8 +38,7 @@ def main(): | |||||
# Create database object. Needs to be serialized before passing | # Create database object. Needs to be serialized before passing | ||||
# to the Server. | # to the Server. | ||||
db = nilmdb.utils.serializer_proxy( | |||||
nilmdb.server.NilmDB)(args.database, sync = not args.nosync) | |||||
db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(args.database) | |||||
# Configure the server | # Configure the server | ||||
if args.quiet: | if args.quiet: | ||||
@@ -31,11 +31,9 @@ import bisect | |||||
# after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA. | # after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA. | ||||
# 3: at the end of an explicit transaction, e.g. "with self.con as con:" | # 3: at the end of an explicit transaction, e.g. "with self.con as con:" | ||||
# | # | ||||
# To speed up testing, or if this transaction speed becomes an issue, | |||||
# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF. | |||||
# Don't touch old entries -- just add new ones. | |||||
# To speed things up, we can set 'PRAGMA synchronous=OFF'. Or, it | |||||
# seems that 'PRAGMA synchronous=NORMAL' and 'PRAGMA journal_mode=WAL' | |||||
# give an equivalent speedup more safely. That is what is used here. | |||||
_sql_schema_updates = { | _sql_schema_updates = { | ||||
0: """ | 0: """ | ||||
-- All streams | -- All streams | ||||
@@ -77,7 +75,7 @@ _sql_schema_updates = { | |||||
class NilmDB(object): | class NilmDB(object): | ||||
verbose = 0 | verbose = 0 | ||||
def __init__(self, basepath, sync=True, max_results=None, | |||||
def __init__(self, basepath, max_results=None, | |||||
bulkdata_args=None): | bulkdata_args=None): | ||||
if bulkdata_args is None: | if bulkdata_args is None: | ||||
bulkdata_args = {} | bulkdata_args = {} | ||||
@@ -101,10 +99,8 @@ class NilmDB(object): | |||||
self._sql_schema_update() | self._sql_schema_update() | ||||
# See big comment at top about the performance implications of this | # See big comment at top about the performance implications of this | ||||
if sync: | |||||
self.con.execute("PRAGMA synchronous=FULL") | |||||
else: | |||||
self.con.execute("PRAGMA synchronous=OFF") | |||||
self.con.execute("PRAGMA synchronous=NORMAL") | |||||
self.con.execute("PRAGMA journal_mode=WAL") | |||||
# Approximate largest number of elements that we want to send | # Approximate largest number of elements that we want to send | ||||
# in a single reply (for stream_intervals, stream_extract) | # in a single reply (for stream_intervals, stream_extract) | ||||
@@ -35,8 +35,7 @@ def setup_module(): | |||||
recursive_unlink(testdb) | recursive_unlink(testdb) | ||||
# Start web app on a custom port | # Start web app on a custom port | ||||
test_db = nilmdb.utils.serializer_proxy( | |||||
nilmdb.server.NilmDB)(testdb, sync = False) | |||||
test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(testdb) | |||||
test_server = nilmdb.server.Server(test_db, host = "127.0.0.1", | test_server = nilmdb.server.Server(test_db, host = "127.0.0.1", | ||||
port = 32180, stoppable = False, | port = 32180, stoppable = False, | ||||
fast_shutdown = True, | fast_shutdown = True, | ||||
@@ -24,7 +24,7 @@ def server_start(max_results = None, bulkdata_args = {}): | |||||
global test_server, test_db | global test_server, test_db | ||||
# Start web app on a custom port | # Start web app on a custom port | ||||
test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)( | test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)( | ||||
testdb, sync = False, | |||||
testdb, | |||||
max_results = max_results, | max_results = max_results, | ||||
bulkdata_args = bulkdata_args) | bulkdata_args = bulkdata_args) | ||||
test_server = nilmdb.server.Server(test_db, host = "127.0.0.1", | test_server = nilmdb.server.Server(test_db, host = "127.0.0.1", | ||||
@@ -33,7 +33,7 @@ class Test00Nilmdb(object): # named 00 so it runs first | |||||
db = nilmdb.server.NilmDB(testdb) | db = nilmdb.server.NilmDB(testdb) | ||||
db.close() | db.close() | ||||
db = nilmdb.server.NilmDB(testdb, sync=False) | |||||
db = nilmdb.server.NilmDB(testdb) | |||||
db.close() | db.close() | ||||
# test timer, just to get coverage | # test timer, just to get coverage | ||||
@@ -46,7 +46,7 @@ class Test00Nilmdb(object): # named 00 so it runs first | |||||
in_("test: ", capture.getvalue()) | in_("test: ", capture.getvalue()) | ||||
def test_stream(self): | def test_stream(self): | ||||
db = nilmdb.server.NilmDB(testdb, sync=False) | |||||
db = nilmdb.server.NilmDB(testdb) | |||||
eq_(db.stream_list(), []) | eq_(db.stream_list(), []) | ||||
# Bad path | # Bad path | ||||
@@ -104,7 +104,7 @@ class Test00Nilmdb(object): # named 00 so it runs first | |||||
class TestBlockingServer(object): | class TestBlockingServer(object): | ||||
def setUp(self): | def setUp(self): | ||||
self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb, sync=False) | |||||
self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb) | |||||
def tearDown(self): | def tearDown(self): | ||||
self.db.close() | self.db.close() | ||||
@@ -145,7 +145,7 @@ class TestServer(object): | |||||
def setUp(self): | def setUp(self): | ||||
# Start web app on a custom port | # Start web app on a custom port | ||||
self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb, sync=False) | |||||
self.db = serializer_proxy(nilmdb.server.NilmDB)(testdb) | |||||
self.server = nilmdb.server.Server(self.db, host = "127.0.0.1", | self.server = nilmdb.server.Server(self.db, host = "127.0.0.1", | ||||
port = 32180, stoppable = False) | port = 32180, stoppable = False) | ||||
self.server.start(blocking = False) | self.server.start(blocking = False) | ||||