You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

267 lines
9.6 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb
  3. from nilmdb.utils.printf import *
  4. from nose.tools import *
  5. from nose.tools import assert_raises
  6. import distutils.version
  7. import itertools
  8. import os
  9. import sys
  10. import random
  11. import unittest
  12. from testutil.helpers import *
  13. from nilmdb.server.layout import *
  14. class TestLayouts(object):
  15. # Some nilmdb.layout tests. Not complete, just fills in missing
  16. # coverage.
  17. def test_layouts(self):
  18. x = nilmdb.server.layout.get_named("float32_8")
  19. y = nilmdb.server.layout.get_named("float32_8")
  20. eq_(x.count, y.count)
  21. eq_(x.datatype, y.datatype)
  22. y = nilmdb.server.layout.get_named("float32_7")
  23. ne_(x.count, y.count)
  24. eq_(x.datatype, y.datatype)
  25. def test_parsing(self):
  26. self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
  27. self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
  28. def real_t_parsing(self, name_prep, name_raw, name_rawnotch):
  29. # invalid layouts
  30. with assert_raises(TypeError) as e:
  31. parser = Parser("NoSuchLayout")
  32. with assert_raises(TypeError) as e:
  33. parser = Parser("float32")
  34. # too little data
  35. parser = Parser(name_prep)
  36. data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5\n" +
  37. "1234567890.100000 1.1 2.2 3.3 4.4 5.5\n")
  38. with assert_raises(ParserError) as e:
  39. parser.parse(data)
  40. in_("error", str(e.exception))
  41. # too much data
  42. parser = Parser(name_prep)
  43. data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n" +
  44. "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n")
  45. with assert_raises(ParserError) as e:
  46. parser.parse(data)
  47. in_("error", str(e.exception))
  48. # just right
  49. parser = Parser(name_prep)
  50. data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n" +
  51. "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n")
  52. parser.parse(data)
  53. eq_(parser.min_timestamp, 1234567890.0)
  54. eq_(parser.max_timestamp, 1234567890.1)
  55. eq_(parser.data, [[1234567890.0,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8],
  56. [1234567890.1,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]])
  57. # try uint16_6 too, with clamping
  58. parser = Parser(name_raw)
  59. data = ( "1234567890.000000 1 2 3 4 5 6\n" +
  60. "1234567890.100000 1 2 3 4 5 6\n" )
  61. parser.parse(data)
  62. eq_(parser.data, [[1234567890.0,1,2,3,4,5,6],
  63. [1234567890.1,1,2,3,4,5,6]])
  64. # pass an instantiated class
  65. parser = Parser(get_named(name_rawnotch))
  66. data = ( "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
  67. "1234567890.100000 1 2 3 4 5 6 7 8 9\n" )
  68. parser.parse(data)
  69. # non-monotonic
  70. parser = Parser(name_raw)
  71. data = ( "1234567890.100000 1 2 3 4 5 6\n" +
  72. "1234567890.099999 1 2 3 4 5 6\n" )
  73. with assert_raises(ParserError) as e:
  74. parser.parse(data)
  75. in_("not monotonically increasing", str(e.exception))
  76. parser = Parser(name_raw)
  77. data = ( "1234567890.100000 1 2 3 4 5 6\n" +
  78. "1234567890.100000 1 2 3 4 5 6\n" )
  79. with assert_raises(ParserError) as e:
  80. parser.parse(data)
  81. in_("not monotonically increasing", str(e.exception))
  82. parser = Parser(name_raw)
  83. data = ( "1234567890.100000 1 2 3 4 5 6\n" +
  84. "1234567890.100001 1 2 3 4 5 6\n" )
  85. parser.parse(data)
  86. # uint16_6 with values out of bounds
  87. parser = Parser(name_raw)
  88. data = ( "1234567890.000000 1 2 3 4 500000 6\n" +
  89. "1234567890.100000 1 2 3 4 5 6\n" )
  90. with assert_raises(ParserError) as e:
  91. parser.parse(data)
  92. in_("value out of range", str(e.exception))
  93. # Empty data should work but is useless
  94. parser = Parser(name_raw)
  95. data = ""
  96. parser.parse(data)
  97. assert(parser.min_timestamp is None)
  98. assert(parser.max_timestamp is None)
  99. def test_formatting(self):
  100. self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
  101. self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
  102. def real_t_formatting(self, name_prep, name_raw, name_rawnotch):
  103. # invalid layout
  104. with assert_raises(TypeError) as e:
  105. formatter = Formatter("NoSuchLayout")
  106. # too little data
  107. formatter = Formatter(name_prep)
  108. data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5 ],
  109. [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5 ] ]
  110. with assert_raises(FormatterError) as e:
  111. formatter.format(data)
  112. in_("error", str(e.exception))
  113. # too much data
  114. formatter = Formatter(name_prep)
  115. data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
  116. [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
  117. with assert_raises(FormatterError) as e:
  118. formatter.format(data)
  119. in_("error", str(e.exception))
  120. # just right
  121. formatter = Formatter(name_prep)
  122. data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ],
  123. [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ] ]
  124. text = formatter.format(data)
  125. eq_(text,
  126. "1234567890.000000 1.100000e+00 2.200000e+00 3.300000e+00 "
  127. "4.400000e+00 5.500000e+00 6.600000e+00 7.700000e+00 "
  128. "8.800000e+00\n" +
  129. "1234567890.100000 1.100000e+00 2.200000e+00 3.300000e+00 "
  130. "4.400000e+00 5.500000e+00 6.600000e+00 7.700000e+00 "
  131. "8.800000e+00\n")
  132. # try uint16_6 too
  133. formatter = Formatter(name_raw)
  134. data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6 ],
  135. [ 1234567890.100000, 1, 2, 3, 4, 5, 6 ] ]
  136. text = formatter.format(data)
  137. eq_(text,
  138. "1234567890.000000 1 2 3 4 5 6\n" +
  139. "1234567890.100000 1 2 3 4 5 6\n")
  140. # pass an instantiated class
  141. formatter = Formatter(get_named(name_rawnotch))
  142. data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
  143. [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
  144. text = formatter.format(data)
  145. eq_(text,
  146. "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
  147. "1234567890.100000 1 2 3 4 5 6 7 8 9\n")
  148. # Empty data should work but is useless
  149. formatter = Formatter(name_raw)
  150. data = []
  151. text = formatter.format(data)
  152. eq_(text, "")
  153. def test_roundtrip(self):
  154. self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
  155. self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
  156. def real_t_roundtrip(self, name_prep, name_raw, name_rawnotch):
  157. # Verify that textual data passed into the Parser, and then
  158. # back through the Formatter, then back into the Parser,
  159. # gives identical parsed representations
  160. random.seed(12345)
  161. def do_roundtrip(layout, datagen):
  162. for i in range(100):
  163. rows = random.randint(1,100)
  164. data = ""
  165. ts = 1234567890
  166. for r in range(rows):
  167. ts += random.uniform(0,1)
  168. row = sprintf("%f", ts) + " "
  169. row += " ".join(datagen())
  170. row += "\n"
  171. data += row
  172. parser1 = Parser(layout)
  173. formatter = Formatter(layout)
  174. parser2 = Parser(layout)
  175. parser1.parse(data)
  176. parser2.parse(formatter.format(parser1.data))
  177. eq_(parser1.data, parser2.data)
  178. def datagen():
  179. return [ sprintf("%.6e", random.uniform(-1000,1000))
  180. for x in range(8) ]
  181. do_roundtrip(name_prep, datagen)
  182. def datagen():
  183. return [ sprintf("%d", random.randint(0,65535))
  184. for x in range(6) ]
  185. do_roundtrip(name_raw, datagen)
  186. def datagen():
  187. return [ sprintf("%d", random.randint(0,65535))
  188. for x in range(9) ]
  189. do_roundtrip(name_rawnotch, datagen)
  190. class TestLayoutSpeed:
  191. @unittest.skip("this is slow")
  192. def test_layout_speed(self):
  193. import time
  194. random.seed(54321)
  195. def do_speedtest(layout, datagen, rows = 5000, times = 100):
  196. # Build data once
  197. data = ""
  198. ts = 1234567890
  199. for r in range(rows):
  200. ts += random.uniform(0,1)
  201. row = sprintf("%f", ts) + " "
  202. row += " ".join(datagen())
  203. row += "\n"
  204. data += row
  205. # Do lots of roundtrips
  206. start = time.time()
  207. for i in range(times):
  208. parser = Parser(layout)
  209. formatter = Formatter(layout)
  210. parser.parse(data)
  211. formatter.format(parser.data)
  212. elapsed = time.time() - start
  213. printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
  214. layout,
  215. elapsed * 1e3,
  216. (elapsed * 1e6) / (rows * times),
  217. (rows * times) / elapsed)
  218. print ""
  219. def datagen():
  220. return [ sprintf("%.6e", random.uniform(-1000,1000))
  221. for x in range(10) ]
  222. do_speedtest("float32_10", datagen)
  223. def datagen():
  224. return [ sprintf("%d", random.randint(0,65535))
  225. for x in range(10) ]
  226. do_speedtest("uint16_10", datagen)
  227. def datagen():
  228. return [ sprintf("%d", random.randint(0,65535))
  229. for x in range(6) ]
  230. do_speedtest("uint16_6", datagen)