Browse Source

More insertion checks. Need to get interval concatenation working.

tags/insert-rework-wip
Jim Paris 9 years ago
parent
commit
32dbeebc09
3 changed files with 44 additions and 9 deletions
  1. +14
    -6
      nilmdb/client.py
  2. +2
    -2
      setup.cfg
  3. +28
    -1
      tests/test_client.py

+ 14
- 6
nilmdb/client.py View File

@@ -19,6 +19,10 @@ from nilmdb.httpclient import ClientError, ServerError, Error

version = "1.0"

def float_to_string(f):
# Use repr to maintain full precision in the string output.
return repr(float(f))

class Client(object):
"""Main client interface to the Nilm database."""

@@ -126,14 +130,18 @@ class Client(object):
# point.
if nextline:
block_end = extract_timestamp(nextline)
if end and block_end > end:
# This is unexpected, but we'll defer to the server
# to return an error in this case.
block_end = end
elif end:
block_end = end
else:
block_end = extract_timestamp(line) + end_epsilon

# Send it
params["start"] = repr(block_start) # use repr to keep precision
params["end"] = repr(block_end)
params["start"] = float_to_string(block_start)
params["end"] = float_to_string(block_end)
return self.http.put("stream/insert", block_data, params)

clock_start = time.time()
@@ -173,9 +181,9 @@ class Client(object):
"path": path
}
if start is not None:
params["start"] = repr(start) # use repr to keep precision
params["start"] = float_to_string(start)
if end is not None:
params["end"] = repr(end)
params["end"] = float_to_string(end)
return self.http.get_gen("stream/intervals", params, retjson = True)

def stream_extract(self, path, start = None, end = None, count = False):
@@ -191,9 +199,9 @@ class Client(object):
"path": path,
}
if start is not None:
params["start"] = repr(start) # use repr to keep precision
params["start"] = float_to_string(start)
if end is not None:
params["end"] = repr(end)
params["end"] = float_to_string(end)
if count:
params["count"] = 1



+ 2
- 2
setup.cfg View File

@@ -2,7 +2,7 @@
# note: the value doesn't matter, that's why they're empty here
nocapture=
nologcapture= # comment to see cherrypy logs on failure
#with-coverage=
with-coverage=
cover-inclusive=
cover-package=nilmdb
cover-erase=
@@ -16,7 +16,7 @@ verbosity=2
#tests=tests/test_interval.py
#tests=tests/test_rbtree.py,tests/test_interval.py
#tests=tests/test_interval.py
tests=tests/test_client.py
#tests=tests/test_client.py
#tests=tests/test_timestamper.py
#tests=tests/test_serializer.py
#tests=tests/test_iteratorizer.py


+ 28
- 1
tests/test_client.py View File

@@ -131,6 +131,7 @@ class TestClient(object):

testfile = "tests/data/prep-20120323T1000"
start = datetime_tz.datetime_tz.smartparse("20120323T1000")
start = start.totimestamp()
rate = 120

# First try a nonexistent path
@@ -160,11 +161,37 @@ class TestClient(object):
in_("400 Bad Request", str(e.exception))
in_("no data provided", str(e.exception))

# Specify start/end (starts too late)
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
with assert_raises(ClientError) as e:
result = client.stream_insert("/newton/prep", data,
start + 5, start + 120)
in_("400 Bad Request", str(e.exception))
in_("Data timestamp 1332511200.0 < start time 1332511205.0",
str(e.exception))

# Specify start/end (ends too early)
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
with assert_raises(ClientError) as e:
result = client.stream_insert("/newton/prep", data,
start, start + 1)
in_("400 Bad Request", str(e.exception))
# Client chunks the input, so the exact timestamp here might change
# if the chunk positions change.
in_("Data timestamp 1332511271.016667 >= end time 1332511201.0",
str(e.exception))

# Now do the real load
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
result = client.stream_insert("/newton/prep", data)
result = client.stream_insert("/newton/prep", data,
start, start + 119.999777)
eq_(result, "ok")

# Verify the intervals. Should be just one, even if the data
# was inserted in chunks, due to nilmdb interval concatenation.
intervals = list(client.stream_intervals("/newton/prep"))
eq_(intervals, [[start, start + 119.999777]])

# Try some overlapping data -- just insert it again
data = nilmdb.timestamper.TimestamperRate(testfile, start, 120)
with assert_raises(ClientError) as e:


Loading…
Cancel
Save