You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

layout.pyx 6.3 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209
  1. # cython: profile=False
  2. import time
  3. import sys
  4. import inspect
  5. import cStringIO
  6. import numpy as np
  7. cdef enum:
  8. max_value_count = 64
  9. cimport cython
  10. cimport libc.stdlib
  11. cimport libc.stdio
  12. cimport libc.string
  13. class ParserError(Exception):
  14. def __init__(self, line, message):
  15. self.message = "line " + str(line) + ": " + message
  16. Exception.__init__(self, self.message)
  17. class FormatterError(Exception):
  18. pass
  19. class Layout:
  20. """Represents a NILM database layout"""
  21. def __init__(self, typestring):
  22. """Initialize this Layout object to handle the specified
  23. type string"""
  24. try:
  25. [ datatype, count ] = typestring.split("_")
  26. except:
  27. raise KeyError("invalid layout string")
  28. try:
  29. self.count = int(count)
  30. except ValueError:
  31. raise KeyError("invalid count")
  32. if self.count < 1 or self.count > max_value_count:
  33. raise KeyError("invalid count")
  34. if datatype == 'uint16':
  35. self.parse = self.parse_uint16
  36. self.format = self.format_uint16
  37. elif datatype == 'float32' or datatype == 'float64':
  38. self.parse = self.parse_float64
  39. self.format = self.format_float64
  40. else:
  41. raise KeyError("invalid type")
  42. self.datatype = datatype
  43. # Parsers
  44. def parse_float64(self, char *text):
  45. cdef int n
  46. cdef double ts
  47. # Return doubles even in float32 case, since they're going into
  48. # a Python array which would upconvert to double anyway.
  49. result = []
  50. cdef char *end
  51. ts = libc.stdlib.strtod(text, &end)
  52. if end == text:
  53. raise ValueError("bad timestamp")
  54. result.append(ts)
  55. for n in range(self.count):
  56. text = end
  57. result.append(libc.stdlib.strtod(text, &end))
  58. if end == text:
  59. raise ValueError("wrong number of values")
  60. n = 0
  61. while end[n] == ' ':
  62. n += 1
  63. if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
  64. raise ValueError("extra data on line")
  65. return (ts, result)
  66. def parse_uint16(self, char *text):
  67. cdef int n
  68. cdef double ts
  69. cdef int v
  70. result = []
  71. cdef char *end
  72. ts = libc.stdlib.strtod(text, &end)
  73. if end == text:
  74. raise ValueError("bad timestamp")
  75. result.append(ts)
  76. for n in range(self.count):
  77. text = end
  78. v = libc.stdlib.strtol(text, &end, 10)
  79. if v < 0 or v > 65535:
  80. raise ValueError("value out of range")
  81. result.append(v)
  82. if end == text:
  83. raise ValueError("wrong number of values")
  84. n = 0
  85. while end[n] == ' ':
  86. n += 1
  87. if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
  88. raise ValueError("extra data on line")
  89. return (ts, result)
  90. # Formatters
  91. def format_float64(self, d):
  92. n = len(d) - 1
  93. if n != self.count:
  94. raise ValueError("wrong number of values for layout type: "
  95. "got %d, wanted %d" % (n, self.count))
  96. s = "%.6f" % d[0]
  97. for i in range(n):
  98. s += " %f" % d[i+1]
  99. return s + "\n"
  100. def format_uint16(self, d):
  101. n = len(d) - 1
  102. if n != self.count:
  103. raise ValueError("wrong number of values for layout type: "
  104. "got %d, wanted %d" % (n, self.count))
  105. s = "%.6f" % d[0]
  106. for i in range(n):
  107. s += " %d" % d[i+1]
  108. return s + "\n"
  109. # Get a layout by name
  110. def get_named(typestring):
  111. try:
  112. return Layout(typestring)
  113. except KeyError:
  114. compat = { "PrepData": "float32_8",
  115. "RawData": "uint16_6",
  116. "RawNotchedData": "uint16_9" }
  117. return Layout(compat[typestring])
  118. class Parser(object):
  119. """Object that parses and stores ASCII data for inclusion into the
  120. database"""
  121. def __init__(self, layout):
  122. if issubclass(layout.__class__, Layout):
  123. self.layout = layout
  124. else:
  125. try:
  126. self.layout = get_named(layout)
  127. except KeyError:
  128. raise TypeError("unknown layout")
  129. self.data = []
  130. self.min_timestamp = None
  131. self.max_timestamp = None
  132. def parse(self, textdata):
  133. """
  134. Parse the data, provided as lines of text, using the current
  135. layout, into an internal data structure suitable for a
  136. pytables 'table.append(parser.data)'.
  137. """
  138. cdef double last_ts = 0, ts
  139. cdef int n = 0, i
  140. cdef char *line
  141. indata = cStringIO.StringIO(textdata)
  142. # Assume any parsing error is a real error.
  143. # In the future we might want to skip completely empty lines,
  144. # or partial lines right before EOF?
  145. try:
  146. self.data = []
  147. for pyline in indata:
  148. line = pyline
  149. n += 1
  150. if line[0] == '\#':
  151. continue
  152. (ts, row) = self.layout.parse(line)
  153. if ts <= last_ts:
  154. raise ValueError("timestamp is not "
  155. "monotonically increasing")
  156. last_ts = ts
  157. self.data.append(row)
  158. except (ValueError, IndexError, TypeError) as e:
  159. raise ParserError(n, "error: " + e.message)
  160. # Mark timestamp ranges
  161. if len(self.data):
  162. self.min_timestamp = self.data[0][0]
  163. self.max_timestamp = self.data[-1][0]
  164. class Formatter(object):
  165. """Object that formats database data into ASCII"""
  166. def __init__(self, layout):
  167. if issubclass(layout.__class__, Layout):
  168. self.layout = layout
  169. else:
  170. try:
  171. self.layout = get_named(layout)
  172. except KeyError:
  173. raise TypeError("unknown layout")
  174. def format(self, data):
  175. """
  176. Format raw data from the database, using the current layout,
  177. as lines of ACSII text.
  178. """
  179. text = cStringIO.StringIO()
  180. try:
  181. for row in data:
  182. text.write(self.layout.format(row))
  183. except (ValueError, IndexError, TypeError) as e:
  184. raise FormatterError("formatting error: " + e.message)
  185. return text.getvalue()