Coalesce calls to table.remove during stream_remove; significant speedup for degenerate cases

This commit is contained in:
Jim Paris 2014-02-14 15:13:17 -05:00
parent 0a22db3965
commit ddc0eb4264

View File

@ -649,6 +649,9 @@ class NilmDB(object):
# remember what's currently in the intersection now.
all_candidates = list(intervals.intersection(to_remove, orig = True))
remove_start = None
remove_end = None
for (dbint, orig) in all_candidates:
# Find row start and end
row_start = self._find_start(table, dbint)
@ -670,8 +673,18 @@ class NilmDB(object):
# Remove interval from the database
self._remove_interval(stream_id, orig, dbint)
# Remove data from the underlying table storage
table.remove(row_start, row_end)
# Remove data from the underlying table storage,
# coalescing adjacent removals to reduce the number of calls
# to table.remove.
if remove_end == row_start:
# Extend our coalesced region
remove_end = row_end
else:
# Perform previous removal, then save this one
if remove_end is not None:
table.remove(remove_start, remove_end)
remove_start = row_start
remove_end = row_end
# Count how many were removed
removed += row_end - row_start
@ -680,4 +693,8 @@ class NilmDB(object):
if restart is not None:
break
# Perform any final coalesced removal
if remove_end is not None:
table.remove(remove_start, remove_end)
return (removed, restart)