Browse Source

Use nilmdb.lrucache for caching interval sets

tags/replace-pytables
Jim Paris 11 years ago
parent
commit
173014ba19
2 changed files with 23 additions and 25 deletions
  1. +3
    -0
      nilmdb/lrucache.py
  2. +20
    -25
      nilmdb/nilmdb.py

+ 3
- 0
nilmdb/lrucache.py View File

@@ -46,6 +46,9 @@ def lrucache(size = 10, onremove = None):
return value

def cache_remove(*args, **kwargs):
"""Remove the described key from this cache, if present.
Note that if the original wrapped function was implicitly
passed 'self', you need to pass it as an argument here too."""
key = args + tuple(sorted(kwargs.items()))
if key in cache:
evict(cache.pop(key))


+ 20
- 25
nilmdb/nilmdb.py View File

@@ -24,6 +24,7 @@ import bisect
import pyximport
pyximport.install()
from nilmdb.interval import Interval, DBInterval, IntervalSet, IntervalError
from nilmdb.lrucache import lrucache

# Note about performance and transactions:
#
@@ -137,9 +138,6 @@ class NilmDB(object):

self.opened = True

# Cached intervals
self._cached_iset = {}

def __del__(self):
if "opened" in self.__dict__: # pragma: no cover
fprintf(sys.stderr,
@@ -171,34 +169,32 @@ class NilmDB(object):
with self.con:
cur.execute("PRAGMA user_version = {v:d}".format(v=version))

@lrucache(size = 16)
def _get_intervals(self, stream_id):
"""
Return a mutable IntervalSet corresponding to the given stream ID.
"""
# Load from database if not cached
if stream_id not in self._cached_iset:
iset = IntervalSet()
result = self.con.execute("SELECT start_time, end_time, "
"start_pos, end_pos "
"FROM ranges "
"WHERE stream_id=?", (stream_id,))
try:
for (start_time, end_time, start_pos, end_pos) in result:
iset += DBInterval(start_time, end_time,
start_time, end_time,
start_pos, end_pos)
except IntervalError as e: # pragma: no cover
raise NilmDBError("unexpected overlap in ranges table!")
self._cached_iset[stream_id] = iset
# Return cached value
return self._cached_iset[stream_id]
iset = IntervalSet()
result = self.con.execute("SELECT start_time, end_time, "
"start_pos, end_pos "
"FROM ranges "
"WHERE stream_id=?", (stream_id,))
try:
for (start_time, end_time, start_pos, end_pos) in result:
iset += DBInterval(start_time, end_time,
start_time, end_time,
start_pos, end_pos)
except IntervalError as e: # pragma: no cover
raise NilmDBError("unexpected overlap in ranges table!")

return iset

def _add_interval(self, stream_id, interval, start_pos, end_pos):
"""
Add interval to the internal interval cache, and to the database.
Note: arguments must be ints (not numpy.int64, etc)
"""
# Ensure this stream's intervals are cached
# Load this stream's intervals
iset = self._get_intervals(stream_id)

# Check for overlap
@@ -216,7 +212,7 @@ class NilmDB(object):
if (adjacent is not None and
start_pos == adjacent.db_endpos and
(end_pos - adjacent.db_startpos) < max_merged_rows):
# First delete the old one, both from our cache and the
# First delete the old one, both from our iset and the
# database
iset -= adjacent
self.con.execute("DELETE FROM ranges WHERE "
@@ -233,7 +229,7 @@ class NilmDB(object):
interval.start = adjacent.start
start_pos = adjacent.db_startpos

# Add the new interval to the cache
# Add the new interval to the iset
iset.iadd_nocheck(DBInterval(interval.start, interval.end,
interval.start, interval.end,
start_pos, end_pos))
@@ -387,8 +383,7 @@ class NilmDB(object):
stream_id = self._stream_id(path)

# Delete the cached interval data
if stream_id in self._cached_iset:
del self._cached_iset[stream_id]
self._get_intervals.cache_remove(self, stream_id)

# Delete the data node, and all parent nodes (if they have no
# remaining children)


Loading…
Cancel
Save