Browse Source

Coalesce calls to table.remove during stream_remove; significant speedup for degenerate cases

tags/nilmdb-1.10.0
Jim Paris 9 years ago
parent
commit
ddc0eb4264
1 changed files with 19 additions and 2 deletions
  1. +19
    -2
      nilmdb/server/nilmdb.py

+ 19
- 2
nilmdb/server/nilmdb.py View File

@@ -649,6 +649,9 @@ class NilmDB(object):
# remember what's currently in the intersection now.
all_candidates = list(intervals.intersection(to_remove, orig = True))

remove_start = None
remove_end = None

for (dbint, orig) in all_candidates:
# Find row start and end
row_start = self._find_start(table, dbint)
@@ -670,8 +673,18 @@ class NilmDB(object):
# Remove interval from the database
self._remove_interval(stream_id, orig, dbint)

# Remove data from the underlying table storage
table.remove(row_start, row_end)
# Remove data from the underlying table storage,
# coalescing adjacent removals to reduce the number of calls
# to table.remove.
if remove_end == row_start:
# Extend our coalesced region
remove_end = row_end
else:
# Perform previous removal, then save this one
if remove_end is not None:
table.remove(remove_start, remove_end)
remove_start = row_start
remove_end = row_end

# Count how many were removed
removed += row_end - row_start
@@ -680,4 +693,8 @@ class NilmDB(object):
if restart is not None:
break

# Perform any final coalesced removal
if remove_end is not None:
table.remove(remove_start, remove_end)

return (removed, restart)

Loading…
Cancel
Save