Browse Source

Fix slow database stuff in testing by setting PRAGMA synchronous=OFF. Default is still FULL for production use, though

git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10639 ddd99763-3ecb-0310-9145-efcb8ce7c51f
tags/bxinterval-last
Jim Paris 10 years ago
parent
commit
913883be3a
4 changed files with 27 additions and 9 deletions
  1. +20
    -2
      nilmdb/nilmdb.py
  2. +2
    -2
      setup.cfg
  3. +1
    -1
      tests/test_client.py
  4. +4
    -4
      tests/test_nilmdb.py

+ 20
- 2
nilmdb/nilmdb.py View File

@@ -20,6 +20,19 @@ import sys
import os
import errno

# Note about performance and transactions:
#
# Committing a transaction in the default sync mode (PRAGMA synchronous=FULL)
# takes about 125msec. sqlite3 will commit transactions at 3 times:
# 1: explicit con.commit()
# 2: between a series of DML commands and non-DML commands, e.g.
# after a series of INSERT, SELECT, but before a CREATE TABLE or PRAGMA.
# 3: at the end of an explicit transaction, e.g. "with self.con as con:"
#
# To speed up testing, or if this transaction speed becomes an issue,
# the sync=False option to NilmDB.__init__ will set PRAGMA synchronous=OFF.


# Don't touch old entries -- just add new ones.
sql_schema_updates = {
0: """
@@ -61,7 +74,7 @@ class StreamError(Exception):
class NilmDB(object):
verbose = 0

def __init__(self, basepath):
def __init__(self, basepath, sync=True):
# set up path
self.basepath = basepath.rstrip('/')

@@ -85,6 +98,12 @@ class NilmDB(object):
self.con = sqlite3.connect(sqlfilename, check_same_thread = False)
self.sql_schema_update()

# See big comment at top about the performance implications of this
if sync:
self.con.execute("PRAGMA synchronous=FULL")
else:
self.con.execute("PRAGMA synchronous=OFF")
self.opened = True

def __del__(self):
@@ -112,7 +131,6 @@ class NilmDB(object):
printf("Schema updated to %d\n", version)

if version != oldversion:
# this takes .1 seconds, so only do it if necessary
with self.con:
cur.execute("PRAGMA user_version = {v:d}".format(v=version))



+ 2
- 2
setup.cfg View File

@@ -10,7 +10,7 @@ cover-erase=1
stop=1
verbosity=2
#tests=tests/test_nilmdb.py
#tests=tests/test_client.py
#tests=tests/test_client.py:TestClient.test_client_nilmdb
#with-profile=1
#profile-sort=time
##profile-restrict=10 # doesn't work right, treated as string or something
##profile-restrict=10 # doesn't work right, treated as string or something

+ 1
- 1
tests/test_client.py View File

@@ -36,7 +36,7 @@ def setup_module():
pass

# Start web app on a custom port
test_db = nilmdb.NilmDB(testdb)
test_db = nilmdb.NilmDB(testdb, sync=False)
test_server = nilmdb.Server(test_db, host = "127.0.0.1",
port = 12380, stoppable = False,
fast_shutdown = True)


+ 4
- 4
tests/test_nilmdb.py View File

@@ -36,11 +36,11 @@ class Test00Nilmdb(object): # named 00 so it runs first

db = nilmdb.NilmDB(testdb)
db.close()
db = nilmdb.NilmDB(testdb)
db = nilmdb.NilmDB(testdb, sync=False)
db.close()

def test_stream(self):
db = nilmdb.NilmDB(testdb)
db = nilmdb.NilmDB(testdb, sync=False)
eq_(db.stream_list(), [])

# Bad path
@@ -92,7 +92,7 @@ class Test00Nilmdb(object): # named 00 so it runs first

class TestBlockingServer(object):
def setUp(self):
self.db = nilmdb.NilmDB(testdb)
self.db = nilmdb.NilmDB(testdb, sync=False)

def tearDown(self):
self.db.close()
@@ -127,7 +127,7 @@ class TestServer(object):

def setUp(self):
# Start web app on a custom port
self.db = nilmdb.NilmDB(testdb)
self.db = nilmdb.NilmDB(testdb, sync=False)
self.server = nilmdb.Server(self.db, host = "127.0.0.1",
port = 12380, stoppable = False)
self.server.start(blocking = False)


Loading…
Cancel
Save