diff --git a/nilmdb/client/numpyclient.py b/nilmdb/client/numpyclient.py new file mode 100644 index 0000000..5b02a97 --- /dev/null +++ b/nilmdb/client/numpyclient.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +"""Provide a NumpyClient class that is based on normal Client, but has +additional methods for extracting and inserting data via Numpy arrays.""" + +import nilmdb.utils +import nilmdb.client.client +import nilmdb.client.httpclient +from nilmdb.client.errors import ClientError + +import contextlib +from nilmdb.utils.time import timestamp_to_string, string_to_timestamp + +import numpy +import cStringIO + +def layout_to_dtype(layout): + ltype = layout.split('_')[0] + lcount = int(layout.split('_')[1]) + if ltype.startswith('int'): + atype = '= maxsize: + assembled = "".join(chunks) + total_len -= maxsize + chunks = [ assembled[maxsize:] ] + block = assembled[:maxsize] + yield to_numpy(block) + + if total_len: + yield to_numpy("".join(chunks)) diff --git a/tests/test.order b/tests/test.order index 59e528e..0b3f06e 100644 --- a/tests/test.order +++ b/tests/test.order @@ -12,6 +12,7 @@ test_interval.py test_bulkdata.py test_nilmdb.py test_client.py +test_numpyclient.py test_cmdline.py test_*.py diff --git a/tests/test_numpyclient.py b/tests/test_numpyclient.py new file mode 100644 index 0000000..915762d --- /dev/null +++ b/tests/test_numpyclient.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import nilmdb.server +import nilmdb.client +import nilmdb.client.numpyclient + +from nilmdb.utils.printf import * +from nilmdb.utils import timestamper +from nilmdb.client import ClientError, ServerError +from nilmdb.utils import datetime_tz + +from nose.plugins.skip import SkipTest +from nose.tools import * +from nose.tools import assert_raises +import itertools +import distutils.version + +from testutil.helpers import * + +import numpy as np + +testdb = "tests/numpyclient-testdb" +testurl = "http://localhost:32180/" + +def setup_module(): + global test_server, test_db + # Clear out DB + recursive_unlink(testdb) + + # Start web app on a custom port + test_db = nilmdb.utils.serializer_proxy(nilmdb.server.NilmDB)(testdb) + test_server = nilmdb.server.Server(test_db, host = "127.0.0.1", + port = 32180, stoppable = False, + fast_shutdown = True, + force_traceback = True) + test_server.start(blocking = False) + +def teardown_module(): + global test_server, test_db + # Close web app + test_server.stop() + test_db.close() + +class TestNumpyClient(object): + + def test_numpyclient_01_basic(self): + # Test basic connection + client = nilmdb.client.numpyclient.NumpyClient(url = testurl) + version = client.version() + eq_(distutils.version.LooseVersion(version), + distutils.version.LooseVersion(test_server.version)) + + # Verify subclassing + assert(isinstance(client, nilmdb.client.Client)) + + # Layouts + for layout in "int8_t", "something_8", "integer_1": + with assert_raises(ValueError): + for x in client.stream_extract_numpy("/foo", layout=layout): + pass + for layout in "int8_1", "uint8_30", "int16_20", "float64_100": + with assert_raises(ClientError) as e: + for x in client.stream_extract_numpy("/foo", layout=layout): + pass + in_("No such stream", str(e.exception)) + + with assert_raises(ClientError) as e: + for x in client.stream_extract_numpy("/foo"): + pass + in_("can't get layout for path", str(e.exception)) + + client.close() + + def test_numpyclient_02_extract(self): + client = nilmdb.client.numpyclient.NumpyClient(url = testurl) + + # Insert some data as text + client.stream_create("/newton/prep", "float32_8") + testfile = "tests/data/prep-20120323T1000" + start = nilmdb.utils.time.parse_time("20120323T1000") + rate = 120 + data = timestamper.TimestamperRate(testfile, start, rate) + result = client.stream_insert("/newton/prep", data, + start, start + 119999777) + + # Extract Numpy arrays + array = None + pieces = 0 + for chunk in client.stream_extract_numpy("/newton/prep", maxrows=1000): + pieces += 1 + if array is not None: + array = np.vstack((array, chunk)) + else: + array = chunk + eq_(array.shape, (14400, 9)) + eq_(pieces, 15) + + # Try structured + s = list(client.stream_extract_numpy("/newton/prep", structured = True)) + assert(np.array_equal(np.c_[s[0]['timestamp'], s[0]['data']], array)) + + # Compare. Will be close but not exact because the conversion + # to and from ASCII was lossy. + data = timestamper.TimestamperRate(testfile, start, rate) + actual = np.fromstring(" ".join(data), sep=' ').reshape(14400, 9) + assert(np.allclose(array, actual)) + + client.close()