- # -*- coding: utf-8 -*-
-
- import nilmdb
-
- from nilmdb.utils.printf import *
-
- from nose.tools import *
- from nose.tools import assert_raises
- import distutils.version
- import itertools
- import os
- import sys
- import random
- import unittest
-
- from testutil.helpers import *
-
- from nilmdb.server.layout import *
-
- class TestLayouts(object):
- # Some nilmdb.layout tests. Not complete, just fills in missing
- # coverage.
- def test_layouts(self):
- x = nilmdb.server.layout.get_named("float32_8")
- y = nilmdb.server.layout.get_named("float32_8")
- eq_(x.count, y.count)
- eq_(x.datatype, y.datatype)
- y = nilmdb.server.layout.get_named("float32_7")
- ne_(x.count, y.count)
- eq_(x.datatype, y.datatype)
-
- def test_parsing(self):
- self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
- self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
- def real_t_parsing(self, name_prep, name_raw, name_rawnotch):
- # invalid layouts
- with assert_raises(TypeError) as e:
- parser = Parser("NoSuchLayout")
- with assert_raises(TypeError) as e:
- parser = Parser("float32")
-
- # too little data
- parser = Parser(name_prep)
- data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5\n" +
- "1234567890.100000 1.1 2.2 3.3 4.4 5.5\n")
- with assert_raises(ParserError) as e:
- parser.parse(data)
- in_("error", str(e.exception))
-
- # too much data
- parser = Parser(name_prep)
- data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n" +
- "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n")
- with assert_raises(ParserError) as e:
- parser.parse(data)
- in_("error", str(e.exception))
-
- # just right
- parser = Parser(name_prep)
- data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n" +
- "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n")
- parser.parse(data)
- eq_(parser.min_timestamp, 1234567890.0)
- eq_(parser.max_timestamp, 1234567890.1)
- eq_(parser.data, [[1234567890.0,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8],
- [1234567890.1,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]])
-
- # try uint16_6 too, with clamping
- parser = Parser(name_raw)
- data = ( "1234567890.000000 1 2 3 4 5 6\n" +
- "1234567890.100000 1 2 3 4 5 6\n" )
- parser.parse(data)
- eq_(parser.data, [[1234567890.0,1,2,3,4,5,6],
- [1234567890.1,1,2,3,4,5,6]])
-
- # pass an instantiated class
- parser = Parser(get_named(name_rawnotch))
- data = ( "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
- "1234567890.100000 1 2 3 4 5 6 7 8 9\n" )
- parser.parse(data)
-
- # non-monotonic
- parser = Parser(name_raw)
- data = ( "1234567890.100000 1 2 3 4 5 6\n" +
- "1234567890.099999 1 2 3 4 5 6\n" )
- with assert_raises(ParserError) as e:
- parser.parse(data)
- in_("not monotonically increasing", str(e.exception))
-
- parser = Parser(name_raw)
- data = ( "1234567890.100000 1 2 3 4 5 6\n" +
- "1234567890.100000 1 2 3 4 5 6\n" )
- with assert_raises(ParserError) as e:
- parser.parse(data)
- in_("not monotonically increasing", str(e.exception))
-
- parser = Parser(name_raw)
- data = ( "1234567890.100000 1 2 3 4 5 6\n" +
- "1234567890.100001 1 2 3 4 5 6\n" )
- parser.parse(data)
-
- # uint16_6 with values out of bounds
- parser = Parser(name_raw)
- data = ( "1234567890.000000 1 2 3 4 500000 6\n" +
- "1234567890.100000 1 2 3 4 5 6\n" )
- with assert_raises(ParserError) as e:
- parser.parse(data)
- in_("value out of range", str(e.exception))
-
- # Empty data should work but is useless
- parser = Parser(name_raw)
- data = ""
- parser.parse(data)
- assert(parser.min_timestamp is None)
- assert(parser.max_timestamp is None)
-
- def test_formatting(self):
- self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
- self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
- def real_t_formatting(self, name_prep, name_raw, name_rawnotch):
- # invalid layout
- with assert_raises(TypeError) as e:
- formatter = Formatter("NoSuchLayout")
-
- # too little data
- formatter = Formatter(name_prep)
- data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5 ],
- [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5 ] ]
- with assert_raises(FormatterError) as e:
- formatter.format(data)
- in_("error", str(e.exception))
-
- # too much data
- formatter = Formatter(name_prep)
- data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
- [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
- with assert_raises(FormatterError) as e:
- formatter.format(data)
- in_("error", str(e.exception))
-
- # just right
- formatter = Formatter(name_prep)
- data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ],
- [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ] ]
- text = formatter.format(data)
- eq_(text,
- "1234567890.000000 1.100000e+00 2.200000e+00 3.300000e+00 "
- "4.400000e+00 5.500000e+00 6.600000e+00 7.700000e+00 "
- "8.800000e+00\n" +
- "1234567890.100000 1.100000e+00 2.200000e+00 3.300000e+00 "
- "4.400000e+00 5.500000e+00 6.600000e+00 7.700000e+00 "
- "8.800000e+00\n")
-
- # try uint16_6 too
- formatter = Formatter(name_raw)
- data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6 ],
- [ 1234567890.100000, 1, 2, 3, 4, 5, 6 ] ]
- text = formatter.format(data)
- eq_(text,
- "1234567890.000000 1 2 3 4 5 6\n" +
- "1234567890.100000 1 2 3 4 5 6\n")
-
- # pass an instantiated class
- formatter = Formatter(get_named(name_rawnotch))
- data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
- [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
- text = formatter.format(data)
- eq_(text,
- "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
- "1234567890.100000 1 2 3 4 5 6 7 8 9\n")
-
- # Empty data should work but is useless
- formatter = Formatter(name_raw)
- data = []
- text = formatter.format(data)
- eq_(text, "")
-
- def test_roundtrip(self):
- self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
- self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
- def real_t_roundtrip(self, name_prep, name_raw, name_rawnotch):
- # Verify that textual data passed into the Parser, and then
- # back through the Formatter, then back into the Parser,
- # gives identical parsed representations
- random.seed(12345)
-
- def do_roundtrip(layout, datagen):
- for i in range(100):
- rows = random.randint(1,100)
- data = ""
- ts = 1234567890
- for r in range(rows):
- ts += random.uniform(0,1)
- row = sprintf("%f", ts) + " "
- row += " ".join(datagen())
- row += "\n"
- data += row
- parser1 = Parser(layout)
- formatter = Formatter(layout)
- parser2 = Parser(layout)
- parser1.parse(data)
- parser2.parse(formatter.format(parser1.data))
- eq_(parser1.data, parser2.data)
-
- def datagen():
- return [ sprintf("%.6e", random.uniform(-1000,1000))
- for x in range(8) ]
- do_roundtrip(name_prep, datagen)
-
- def datagen():
- return [ sprintf("%d", random.randint(0,65535))
- for x in range(6) ]
- do_roundtrip(name_raw, datagen)
-
- def datagen():
- return [ sprintf("%d", random.randint(0,65535))
- for x in range(9) ]
- do_roundtrip(name_rawnotch, datagen)
-
- class TestLayoutSpeed:
- @unittest.skip("this is slow")
- def test_layout_speed(self):
- import time
-
- random.seed(54321)
-
- def do_speedtest(layout, datagen, rows = 5000, times = 100):
- # Build data once
- data = ""
- ts = 1234567890
- for r in range(rows):
- ts += random.uniform(0,1)
- row = sprintf("%f", ts) + " "
- row += " ".join(datagen())
- row += "\n"
- data += row
-
- # Do lots of roundtrips
- start = time.time()
- for i in range(times):
- parser = Parser(layout)
- formatter = Formatter(layout)
- parser.parse(data)
- formatter.format(parser.data)
- elapsed = time.time() - start
- printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
- layout,
- elapsed * 1e3,
- (elapsed * 1e6) / (rows * times),
- (rows * times) / elapsed)
-
- print ""
- def datagen():
- return [ sprintf("%.6e", random.uniform(-1000,1000))
- for x in range(10) ]
- do_speedtest("float32_10", datagen)
-
- def datagen():
- return [ sprintf("%d", random.randint(0,65535))
- for x in range(10) ]
- do_speedtest("uint16_10", datagen)
-
- def datagen():
- return [ sprintf("%d", random.randint(0,65535))
- for x in range(6) ]
- do_speedtest("uint16_6", datagen)
|