You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

190 lines
6.3 KiB

  1. # cython: profile=False
  2. import tables
  3. import time
  4. import sys
  5. import inspect
  6. import cStringIO
  7. import numpy as np
  8. cimport cython
  9. cimport libc.stdlib
  10. cimport libc.stdio
  11. cimport libc.string
  12. class ParserError(Exception):
  13. def __init__(self, line, message):
  14. self.message = "line " + str(line) + ": " + message
  15. Exception.__init__(self, self.message)
  16. class FormatterError(Exception):
  17. pass
  18. class Layout:
  19. """Represents a NILM database layout"""
  20. def description(self):
  21. """Return the PyTables description of this layout"""
  22. desc = {}
  23. for (n, (name, type)) in enumerate(self.fields):
  24. desc[name] = tables.Col.from_type(type, pos=n+1)
  25. return tables.Description(desc)
  26. def parse(self, char *text):
  27. raise ParserError("no parser for this layout")
  28. class PrepData(Layout):
  29. rate_hz = 120
  30. fields = [ ( 'timestamp', 'float64' ),
  31. ( 'p1', 'float32' ),
  32. ( 'q1', 'float32' ),
  33. ( 'p3', 'float32' ),
  34. ( 'q3', 'float32' ),
  35. ( 'p5', 'float32' ),
  36. ( 'q5', 'float32' ),
  37. ( 'p7', 'float32' ),
  38. ( 'q7', 'float32' ) ]
  39. def parse(self, char *text):
  40. cdef int n
  41. cdef double ts
  42. # return doubles instead of float32, since they're going into
  43. # a Python array which would upconvert to double anyway.
  44. cdef double v[8]
  45. cdef char dummy
  46. n = libc.stdio.sscanf(text, " %lf %lf %lf %lf %lf %lf %lf %lf %lf %c",
  47. &ts, &v[0], &v[1], &v[2], &v[3], &v[4],
  48. &v[5], &v[6], &v[7], &dummy)
  49. if (n < 9) or (n > 9 and (dummy != '#' and dummy != '\n')):
  50. raise ValueError("wrong number of values: wanted 9, got " + str(n))
  51. return (ts, [ts, v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7]])
  52. class RawData(Layout):
  53. rate_hz = 8000
  54. fields = [ ( 'timestamp', 'float64' ),
  55. ( 'va', 'uint16' ),
  56. ( 'vb', 'uint16' ),
  57. ( 'vc', 'uint16' ),
  58. ( 'ia', 'uint16' ),
  59. ( 'ib', 'uint16' ),
  60. ( 'ic', 'uint16' ) ]
  61. def parse(self, char *text):
  62. cdef int n
  63. cdef double ts
  64. cdef int v[6]
  65. cdef char dummy
  66. n = libc.stdio.sscanf(text, " %lf %u %u %u %u %u %u %c",
  67. &ts, &v[0], &v[1], &v[2],
  68. &v[3], &v[4], &v[5], &dummy)
  69. if (n < 7) or (n > 7 and (dummy != '#' and dummy != '\n')):
  70. raise ValueError("wrong number of values: wanted 7, got " + str(n))
  71. for i in range(6):
  72. if v[i] < 0 or v[i] > 65535:
  73. raise ValueError("value out of range: " + str(v[i]))
  74. return (ts, [ts, v[0], v[1], v[2], v[3], v[4], v[5]])
  75. class RawNotchedData(RawData):
  76. rate_hz = 8000
  77. fields = RawData.fields + [
  78. ( 'notch_ia', 'uint16' ),
  79. ( 'notch_ib', 'uint16' ),
  80. ( 'notch_ic', 'uint16' ) ]
  81. def parse(self, char *text):
  82. cdef int n
  83. cdef double ts
  84. cdef int v[9]
  85. cdef char dummy
  86. n = libc.stdio.sscanf(text, " %lf %u %u %u %u %u %u %u %u %u %c",
  87. &ts, &v[0], &v[1], &v[2], &v[3], &v[4],
  88. &v[5], &v[6], &v[7], &v[8], &dummy)
  89. if (n < 10) or (n > 10 and (dummy != '#' and dummy != '\n')):
  90. raise ValueError("wrong number of values: wanted 10, got " + str(n))
  91. for i in range(9):
  92. if v[i] < 0 or v[i] > 65535:
  93. raise ValueError("value out of range: " + str(v[i]))
  94. return (ts, [ts, v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8]])
  95. # Instantiate all layouts, indexed by their name
  96. named = {}
  97. for name, obj in inspect.getmembers(sys.modules[__name__]):
  98. if inspect.isclass(obj) and issubclass(obj, Layout):
  99. named[name] = obj()
  100. class Parser(object):
  101. """Object that parses and stores ASCII data for inclusion into the
  102. database"""
  103. def __init__(self, layout):
  104. if issubclass(layout.__class__, Layout):
  105. self.layout = layout
  106. else:
  107. try:
  108. self.layout = named[layout]
  109. except KeyError:
  110. raise TypeError("unknown layout")
  111. self.data = []
  112. self.min_timestamp = None
  113. self.max_timestamp = None
  114. def parse(self, textdata):
  115. """
  116. Parse the data, provided as lines of text, using the current
  117. layout, into an internal data structure suitable for a
  118. pytables 'table.append(parser.data)'.
  119. """
  120. cdef double last_ts = 0, ts
  121. cdef int n = 0, i
  122. cdef char *line
  123. indata = cStringIO.StringIO(textdata)
  124. # Assume any parsing error is a real error.
  125. # In the future we might want to skip completely empty lines,
  126. # or partial lines right before EOF?
  127. try:
  128. self.data = []
  129. for pyline in indata:
  130. line = pyline
  131. n += 1
  132. if line[0] == '\#':
  133. continue
  134. (ts, row) = self.layout.parse(line)
  135. if ts < last_ts:
  136. raise ValueError("timestamp is not "
  137. "monotonically increasing")
  138. last_ts = ts
  139. self.data.append(row)
  140. except (ValueError, IndexError, TypeError) as e:
  141. raise ParserError(n, "error: " + e.message)
  142. # Mark timestamp ranges
  143. if len(self.data):
  144. self.min_timestamp = self.data[0][0]
  145. self.max_timestamp = self.data[-1][0]
  146. class Formatter(object):
  147. """Object that formats database data into ASCII"""
  148. def __init__(self, layout):
  149. if issubclass(layout.__class__, Layout):
  150. self.layout = layout
  151. else:
  152. try:
  153. self.layout = named[layout]
  154. except KeyError:
  155. raise TypeError("unknown layout")
  156. def format(self, data):
  157. """
  158. Format raw data from the database, using the current layout,
  159. as lines of ACSII text.
  160. """
  161. text = cStringIO.StringIO()
  162. try:
  163. for row in data:
  164. text.write(self.layout.format(row))
  165. except (ValueError, IndexError, TypeError) as e:
  166. raise FormatterError("formatting error: " + e.message)
  167. return text.getvalue()