You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

205 lines
6.3 KiB

  1. # cython: profile=False
  2. import time
  3. import sys
  4. import inspect
  5. import cStringIO
  6. from ..utils.time import min_timestamp as nilmdb_min_timestamp
  7. cdef enum:
  8. max_value_count = 64
  9. cimport cython
  10. cimport libc.stdlib
  11. cimport libc.stdio
  12. cimport libc.string
  13. class ParserError(Exception):
  14. def __init__(self, line, message):
  15. self.message = "line " + str(line) + ": " + message
  16. Exception.__init__(self, self.message)
  17. class FormatterError(Exception):
  18. pass
  19. class Layout:
  20. """Represents a NILM database layout"""
  21. def __init__(self, typestring):
  22. """Initialize this Layout object to handle the specified
  23. type string"""
  24. try:
  25. [ datatype, count ] = typestring.split("_")
  26. except:
  27. raise KeyError("invalid layout string")
  28. try:
  29. self.count = int(count)
  30. except ValueError:
  31. raise KeyError("invalid count")
  32. if self.count < 1 or self.count > max_value_count:
  33. raise KeyError("invalid count")
  34. if datatype == 'uint16':
  35. self.parse = self.parse_uint16
  36. self.format_str = "%.6f" + " %d" * self.count
  37. self.format = self.format_generic
  38. elif datatype == 'float32':
  39. self.parse = self.parse_float64
  40. self.format_str = "%.6f" + " %.6e" * self.count
  41. self.format = self.format_generic
  42. elif datatype == 'float64':
  43. self.parse = self.parse_float64
  44. self.format_str = "%.6f" + " %.16e" * self.count
  45. self.format = self.format_generic
  46. else:
  47. raise KeyError("invalid type")
  48. self.datatype = datatype
  49. # Parsers
  50. def parse_float64(self, char *text):
  51. cdef int n
  52. cdef double ts
  53. # Return doubles even in float32 case, since they're going into
  54. # a Python array which would upconvert to double anyway.
  55. result = [0] * (self.count + 1)
  56. cdef char *end
  57. ts = libc.stdlib.strtod(text, &end)
  58. if end == text:
  59. raise ValueError("bad timestamp")
  60. result[0] = ts
  61. for n in range(self.count):
  62. text = end
  63. result[n+1] = libc.stdlib.strtod(text, &end)
  64. if end == text:
  65. raise ValueError("wrong number of values")
  66. n = 0
  67. while end[n] == ' ':
  68. n += 1
  69. if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
  70. raise ValueError("extra data on line")
  71. return (ts, result)
  72. def parse_uint16(self, char *text):
  73. cdef int n
  74. cdef double ts
  75. cdef int v
  76. cdef char *end
  77. result = [0] * (self.count + 1)
  78. ts = libc.stdlib.strtod(text, &end)
  79. if end == text:
  80. raise ValueError("bad timestamp")
  81. result[0] = ts
  82. for n in range(self.count):
  83. text = end
  84. v = libc.stdlib.strtol(text, &end, 10)
  85. if v < 0 or v > 65535:
  86. raise ValueError("value out of range")
  87. result[n+1] = v
  88. if end == text:
  89. raise ValueError("wrong number of values")
  90. n = 0
  91. while end[n] == ' ':
  92. n += 1
  93. if end[n] != '\n' and end[n] != '#' and end[n] != '\0':
  94. raise ValueError("extra data on line")
  95. return (ts, result)
  96. # Formatters
  97. def format_generic(self, d):
  98. n = len(d) - 1
  99. if n != self.count:
  100. raise ValueError("wrong number of values for layout type: "
  101. "got %d, wanted %d" % (n, self.count))
  102. return (self.format_str % tuple(d)) + "\n"
  103. # Get a layout by name
  104. def get_named(typestring):
  105. try:
  106. return Layout(typestring)
  107. except KeyError:
  108. compat = { "PrepData": "float32_8",
  109. "RawData": "uint16_6",
  110. "RawNotchedData": "uint16_9" }
  111. return Layout(compat[typestring])
  112. class Parser(object):
  113. """Object that parses and stores ASCII data for inclusion into the
  114. database"""
  115. def __init__(self, layout):
  116. if issubclass(layout.__class__, Layout):
  117. self.layout = layout
  118. else:
  119. try:
  120. self.layout = get_named(layout)
  121. except KeyError:
  122. raise TypeError("unknown layout")
  123. self.data = []
  124. self.min_timestamp = None
  125. self.max_timestamp = None
  126. def parse(self, textdata):
  127. """
  128. Parse the data, provided as lines of text, using the current
  129. layout, into an internal data structure suitable for a
  130. pytables 'table.append(parser.data)'.
  131. """
  132. cdef double last_ts = nilmdb_min_timestamp
  133. cdef double ts
  134. cdef int n = 0, i
  135. cdef char *line
  136. indata = cStringIO.StringIO(textdata)
  137. # Assume any parsing error is a real error.
  138. # In the future we might want to skip completely empty lines,
  139. # or partial lines right before EOF?
  140. try:
  141. self.data = []
  142. for pyline in indata:
  143. line = pyline
  144. n += 1
  145. if line[0] == '\#':
  146. continue
  147. (ts, row) = self.layout.parse(line)
  148. if ts <= last_ts:
  149. raise ValueError("timestamp is not "
  150. "monotonically increasing")
  151. last_ts = ts
  152. self.data.append(row)
  153. except (ValueError, IndexError, TypeError) as e:
  154. raise ParserError(n, "error: " + e.message)
  155. # Mark timestamp ranges
  156. if len(self.data):
  157. self.min_timestamp = self.data[0][0]
  158. self.max_timestamp = self.data[-1][0]
  159. class Formatter(object):
  160. """Object that formats database data into ASCII"""
  161. def __init__(self, layout):
  162. if issubclass(layout.__class__, Layout):
  163. self.layout = layout
  164. else:
  165. try:
  166. self.layout = get_named(layout)
  167. except KeyError:
  168. raise TypeError("unknown layout")
  169. def format(self, data):
  170. """
  171. Format raw data from the database, using the current layout,
  172. as lines of ACSII text.
  173. """
  174. text = cStringIO.StringIO()
  175. try:
  176. for row in data:
  177. text.write(self.layout.format(row))
  178. except (ValueError, IndexError, TypeError) as e:
  179. raise FormatterError("formatting error: " + e.message)
  180. return text.getvalue()