You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

267 lines
9.5 KiB

  1. # -*- coding: utf-8 -*-
  2. import nilmdb
  3. from nilmdb.utils.printf import *
  4. from nose.tools import *
  5. from nose.tools import assert_raises
  6. import distutils.version
  7. import itertools
  8. import os
  9. import shutil
  10. import sys
  11. import cherrypy
  12. import threading
  13. import urllib2
  14. from urllib2 import urlopen, HTTPError
  15. import Queue
  16. import cStringIO
  17. import random
  18. import unittest
  19. from testutil.helpers import *
  20. from nilmdb.server.layout import *
  21. class TestLayouts(object):
  22. # Some nilmdb.layout tests. Not complete, just fills in missing
  23. # coverage.
  24. def test_layouts(self):
  25. x = nilmdb.server.layout.get_named("PrepData")
  26. y = nilmdb.server.layout.get_named("float32_8")
  27. eq_(x.count, y.count)
  28. eq_(x.datatype, y.datatype)
  29. y = nilmdb.server.layout.get_named("float32_7")
  30. ne_(x.count, y.count)
  31. eq_(x.datatype, y.datatype)
  32. def test_parsing(self):
  33. self.real_t_parsing("PrepData", "RawData", "RawNotchedData")
  34. self.real_t_parsing("float32_8", "uint16_6", "uint16_9")
  35. def real_t_parsing(self, name_prep, name_raw, name_rawnotch):
  36. # invalid layouts
  37. with assert_raises(TypeError) as e:
  38. parser = Parser("NoSuchLayout")
  39. with assert_raises(TypeError) as e:
  40. parser = Parser("float32")
  41. # too little data
  42. parser = Parser(name_prep)
  43. data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5\n" +
  44. "1234567890.100000 1.1 2.2 3.3 4.4 5.5\n")
  45. with assert_raises(ParserError) as e:
  46. parser.parse(data)
  47. in_("error", str(e.exception))
  48. # too much data
  49. parser = Parser(name_prep)
  50. data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n" +
  51. "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8 9.9\n")
  52. with assert_raises(ParserError) as e:
  53. parser.parse(data)
  54. in_("error", str(e.exception))
  55. # just right
  56. parser = Parser(name_prep)
  57. data = ( "1234567890.000000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n" +
  58. "1234567890.100000 1.1 2.2 3.3 4.4 5.5 6.6 7.7 8.8\n")
  59. parser.parse(data)
  60. eq_(parser.min_timestamp, 1234567890.0)
  61. eq_(parser.max_timestamp, 1234567890.1)
  62. eq_(parser.data, [[1234567890.0,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8],
  63. [1234567890.1,1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]])
  64. # try RawData too, with clamping
  65. parser = Parser(name_raw)
  66. data = ( "1234567890.000000 1 2 3 4 5 6\n" +
  67. "1234567890.100000 1 2 3 4 5 6\n" )
  68. parser.parse(data)
  69. eq_(parser.data, [[1234567890.0,1,2,3,4,5,6],
  70. [1234567890.1,1,2,3,4,5,6]])
  71. # pass an instantiated class
  72. parser = Parser(get_named(name_rawnotch))
  73. data = ( "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
  74. "1234567890.100000 1 2 3 4 5 6 7 8 9\n" )
  75. parser.parse(data)
  76. # non-monotonic
  77. parser = Parser(name_raw)
  78. data = ( "1234567890.100000 1 2 3 4 5 6\n" +
  79. "1234567890.099999 1 2 3 4 5 6\n" )
  80. with assert_raises(ParserError) as e:
  81. parser.parse(data)
  82. in_("not monotonically increasing", str(e.exception))
  83. parser = Parser(name_raw)
  84. data = ( "1234567890.100000 1 2 3 4 5 6\n" +
  85. "1234567890.100000 1 2 3 4 5 6\n" )
  86. with assert_raises(ParserError) as e:
  87. parser.parse(data)
  88. in_("not monotonically increasing", str(e.exception))
  89. parser = Parser(name_raw)
  90. data = ( "1234567890.100000 1 2 3 4 5 6\n" +
  91. "1234567890.100001 1 2 3 4 5 6\n" )
  92. parser.parse(data)
  93. # RawData with values out of bounds
  94. parser = Parser(name_raw)
  95. data = ( "1234567890.000000 1 2 3 4 500000 6\n" +
  96. "1234567890.100000 1 2 3 4 5 6\n" )
  97. with assert_raises(ParserError) as e:
  98. parser.parse(data)
  99. in_("value out of range", str(e.exception))
  100. # Empty data should work but is useless
  101. parser = Parser(name_raw)
  102. data = ""
  103. parser.parse(data)
  104. assert(parser.min_timestamp is None)
  105. assert(parser.max_timestamp is None)
  106. def test_formatting(self):
  107. self.real_t_formatting("PrepData", "RawData", "RawNotchedData")
  108. self.real_t_formatting("float32_8", "uint16_6", "uint16_9")
  109. def real_t_formatting(self, name_prep, name_raw, name_rawnotch):
  110. # invalid layout
  111. with assert_raises(TypeError) as e:
  112. formatter = Formatter("NoSuchLayout")
  113. # too little data
  114. formatter = Formatter(name_prep)
  115. data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5 ],
  116. [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5 ] ]
  117. with assert_raises(FormatterError) as e:
  118. formatter.format(data)
  119. in_("error", str(e.exception))
  120. # too much data
  121. formatter = Formatter(name_prep)
  122. data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
  123. [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
  124. with assert_raises(FormatterError) as e:
  125. formatter.format(data)
  126. in_("error", str(e.exception))
  127. # just right
  128. formatter = Formatter(name_prep)
  129. data = [ [ 1234567890.000000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ],
  130. [ 1234567890.100000, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8 ] ]
  131. text = formatter.format(data)
  132. eq_(text,
  133. "1234567890.000000 1.100000 2.200000 3.300000 4.400000 " +
  134. "5.500000 6.600000 7.700000 8.800000\n" +
  135. "1234567890.100000 1.100000 2.200000 3.300000 4.400000 " +
  136. "5.500000 6.600000 7.700000 8.800000\n")
  137. # try RawData too
  138. formatter = Formatter(name_raw)
  139. data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6 ],
  140. [ 1234567890.100000, 1, 2, 3, 4, 5, 6 ] ]
  141. text = formatter.format(data)
  142. eq_(text,
  143. "1234567890.000000 1 2 3 4 5 6\n" +
  144. "1234567890.100000 1 2 3 4 5 6\n")
  145. # pass an instantiated class
  146. formatter = Formatter(get_named(name_rawnotch))
  147. data = [ [ 1234567890.000000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ],
  148. [ 1234567890.100000, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ]
  149. text = formatter.format(data)
  150. eq_(text,
  151. "1234567890.000000 1 2 3 4 5 6 7 8 9\n" +
  152. "1234567890.100000 1 2 3 4 5 6 7 8 9\n")
  153. # Empty data should work but is useless
  154. formatter = Formatter(name_raw)
  155. data = []
  156. text = formatter.format(data)
  157. eq_(text, "")
  158. def test_roundtrip(self):
  159. self.real_t_roundtrip("PrepData", "RawData", "RawNotchedData")
  160. self.real_t_roundtrip("float32_8", "uint16_6", "uint16_9")
  161. def real_t_roundtrip(self, name_prep, name_raw, name_rawnotch):
  162. # Verify that textual data passed into the Parser, and then
  163. # back through the Formatter, then back into the Parser,
  164. # gives identical parsed representations
  165. random.seed(12345)
  166. def do_roundtrip(layout, datagen):
  167. for i in range(100):
  168. rows = random.randint(1,100)
  169. data = ""
  170. ts = 1234567890
  171. for r in range(rows):
  172. ts += random.uniform(0,1)
  173. row = sprintf("%f", ts) + " "
  174. row += " ".join(datagen())
  175. row += "\n"
  176. data += row
  177. parser1 = Parser(layout)
  178. formatter = Formatter(layout)
  179. parser2 = Parser(layout)
  180. parser1.parse(data)
  181. parser2.parse(formatter.format(parser1.data))
  182. eq_(parser1.data, parser2.data)
  183. def datagen():
  184. return [ sprintf("%f", random.uniform(-1000,1000))
  185. for x in range(8) ]
  186. do_roundtrip(name_prep, datagen)
  187. def datagen():
  188. return [ sprintf("%d", random.randint(0,65535))
  189. for x in range(6) ]
  190. do_roundtrip(name_raw, datagen)
  191. def datagen():
  192. return [ sprintf("%d", random.randint(0,65535))
  193. for x in range(9) ]
  194. do_roundtrip(name_rawnotch, datagen)
  195. class TestLayoutSpeed:
  196. @unittest.skip("this is slow")
  197. def test_layout_speed(self):
  198. import time
  199. random.seed(54321)
  200. def do_speedtest(layout, datagen, rows = 5000, times = 100):
  201. # Build data once
  202. data = ""
  203. ts = 1234567890
  204. for r in range(rows):
  205. ts += random.uniform(0,1)
  206. row = sprintf("%f", ts) + " "
  207. row += " ".join(datagen())
  208. row += "\n"
  209. data += row
  210. # Do lots of roundtrips
  211. start = time.time()
  212. for i in range(times):
  213. parser = Parser(layout)
  214. formatter = Formatter(layout)
  215. parser.parse(data)
  216. data = formatter.format(parser.data)
  217. elapsed = time.time() - start
  218. printf("roundtrip %s: %d ms, %.1f μs/row, %d rows/sec\n",
  219. layout,
  220. elapsed * 1e3,
  221. (elapsed * 1e6) / (rows * times),
  222. (rows * times) / elapsed)
  223. print ""
  224. def datagen():
  225. return [ sprintf("%f", random.uniform(-1000,1000))
  226. for x in range(10) ]
  227. do_speedtest("float32_10", datagen)
  228. def datagen():
  229. return [ sprintf("%d", random.randint(0,65535))
  230. for x in range(10) ]
  231. do_speedtest("uint16_10", datagen)