nilmdb/nilmdb/server/rocket.c

#include <Python.h>
#include <structmember.h>
#include <endian.h>

#include <ctype.h>
#include <stdint.h>

/* Values missing from stdint.h */
#define UINT8_MIN 0
#define UINT16_MIN 0
#define UINT32_MIN 0
#define UINT64_MIN 0

/* Marker values (if min == max, skip range check) */
#define FLOAT32_MIN 0
#define FLOAT32_MAX 0
#define FLOAT64_MIN 0
#define FLOAT64_MAX 0

typedef int64_t timestamp_t;

/* This code probably needs to be double-checked for the case
   where sizeof(long) != 8, so enforce that here with something
   that will fail at build time. */
const static char __long_ok[1 - 2*!(sizeof(int64_t) == sizeof(long))] = { 0 };

/* Somewhat arbitrary, just so we can use fixed sizes for strings
   etc. */
static const int MAX_LAYOUT_COUNT = 128;

/* Error object and constants */
static PyObject *ParseError;
typedef enum {
	ERR_OTHER,
	ERR_NON_MONOTONIC,
	ERR_OUT_OF_INTERVAL,
} parseerror_code_t;
static void add_parseerror_codes(PyObject *module)
{
	PyModule_AddIntMacro(module, ERR_OTHER);
	PyModule_AddIntMacro(module, ERR_NON_MONOTONIC);
	PyModule_AddIntMacro(module, ERR_OUT_OF_INTERVAL);
}

/* Helpers to raise ParseErrors.  Use "return raise_str(...)" etc. */
static PyObject *raise_str(int line, int col, int code, const char *string)
{
	PyObject *o;
	o = Py_BuildValue("(iiis)", line, col, code, string);
	if (o != NULL) {
		PyErr_SetObject(ParseError, o);
		Py_DECREF(o);
	}
	return NULL;
}
static PyObject *raise_int(int line, int col, int code, int64_t num)
{
	PyObject *o;
	o = Py_BuildValue("(iiiL)", line, col, code, num);
	if (o != NULL) {
		PyErr_SetObject(ParseError, o);
		Py_DECREF(o);
	}
	return NULL;
}

/****
 * Layout and type helpers
 */
typedef union {
	int8_t i;
	uint8_t u;
} union8_t;
typedef union {
	int16_t i;
	uint16_t u;
} union16_t;
typedef union {
	int32_t i;
	uint32_t u;
	float f;
} union32_t;
typedef union {
	int64_t i;
	uint64_t u;
	double d;
} union64_t;

typedef enum {
	LAYOUT_TYPE_NONE,
	LAYOUT_TYPE_INT8,
	LAYOUT_TYPE_UINT8,
	LAYOUT_TYPE_INT16,
	LAYOUT_TYPE_UINT16,
	LAYOUT_TYPE_INT32,
	LAYOUT_TYPE_UINT32,
	LAYOUT_TYPE_INT64,
	LAYOUT_TYPE_UINT64,
	LAYOUT_TYPE_FLOAT32,
	LAYOUT_TYPE_FLOAT64,
} layout_type_t;

struct {
	char *string;
	layout_type_t layout;
	int size;
} type_lookup[] = {
	{ "int8",    LAYOUT_TYPE_INT8,    1 },
	{ "uint8",   LAYOUT_TYPE_UINT8,   1 },
	{ "int16",   LAYOUT_TYPE_INT16,   2 },
	{ "uint16",  LAYOUT_TYPE_UINT16,  2 },
	{ "int32",   LAYOUT_TYPE_INT32,   4 },
	{ "uint32",  LAYOUT_TYPE_UINT32,  4 },
	{ "int64",   LAYOUT_TYPE_INT64,   8 },
	{ "uint64",  LAYOUT_TYPE_UINT64,  8 },
	{ "float32", LAYOUT_TYPE_FLOAT32, 4 },
	{ "float64", LAYOUT_TYPE_FLOAT64, 8 },
	{ NULL }
};

/****
 * Object definition, init, etc
 */

/* Rocket object */
typedef struct {
	PyObject_HEAD
	layout_type_t layout_type;
	int layout_count;
	int binary_size;
	FILE *file;
	int file_size;
} Rocket;

/* Dealloc / new */
static void Rocket_dealloc(Rocket *self)
{
	if (self->file) {
		fprintf(stderr, "rocket: file wasn't closed\n");
		fclose(self->file);
		self->file = NULL;
	}
	self->ob_type->tp_free((PyObject *)self);
}

static PyObject *Rocket_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
	Rocket *self;

	self = (Rocket *)type->tp_alloc(type, 0);
	if (!self)
		return NULL;
	self->layout_type = LAYOUT_TYPE_NONE;
	self->layout_count = 0;
	self->binary_size = 0;
	self->file = NULL;
	self->file_size = -1;
	return (PyObject *)self;
}

/* .__init__(layout, file) */
static int Rocket_init(Rocket *self, PyObject *args, PyObject *kwds)
{
	const char *layout, *path;
	static char *kwlist[] = { "layout", "file", NULL };
	if (!PyArg_ParseTupleAndKeywords(args, kwds, "sz", kwlist,
					 &layout, &path))
		return -1;
	if (!layout)
		return -1;
	if (path) {
		if ((self->file = fopen(path, "a+b")) == NULL) {
			PyErr_SetFromErrno(PyExc_OSError);
			return -1;
		}
		self->file_size = -1;
	} else {
		self->file = NULL;
	}

	const char *under;
	char *tmp;
	under = strchr(layout, '_');
	if (!under) {
		PyErr_SetString(PyExc_ValueError, "no such layout: "
				"badly formatted string");
		return -1;
	}
	self->layout_count = strtoul(under+1, &tmp, 10);
	if (self->layout_count < 1 || *tmp != '\0') {
		PyErr_SetString(PyExc_ValueError, "no such layout: "
				"bad count");
		return -1;
	}
	if (self->layout_count >= MAX_LAYOUT_COUNT) {
		PyErr_SetString(PyExc_ValueError, "no such layout: "
				"count too high");
		return -1;
	}

	int i;
	for (i = 0; type_lookup[i].string; i++)
		if (strncmp(layout, type_lookup[i].string, under-layout) == 0)
			break;
	if (!type_lookup[i].string) {
		PyErr_SetString(PyExc_ValueError, "no such layout: "
				"bad data type");
		return -1;
	}
	self->layout_type = type_lookup[i].layout;
	self->binary_size = 8 + (type_lookup[i].size * self->layout_count);

	return 0;
}

/* .close() */
static PyObject *Rocket_close(Rocket *self)
{
	if (self->file) {
		fclose(self->file);
		self->file = NULL;
	}
	Py_INCREF(Py_None);
	return Py_None;
}

/* .file_size property */
static PyObject *Rocket_get_file_size(Rocket *self)
{
	if (!self->file) {
		PyErr_SetString(PyExc_AttributeError, "no file");
		return NULL;
	}
	if (self->file_size < 0) {
		int oldpos;
		if (((oldpos = ftell(self->file)) < 0) ||
		    (fseek(self->file, 0, SEEK_END) < 0) ||
		    ((self->file_size = ftell(self->file)) < 0) ||
		    (fseek(self->file, oldpos, SEEK_SET) < 0)) {
			PyErr_SetFromErrno(PyExc_OSError);
			return NULL;
		}
	}
	return PyInt_FromLong(self->file_size);
}

/****
 * Append from iterator
 */

/* Helper for writing Python objects to the file */
static inline void append_pyobject(FILE *out, PyObject *val, layout_type_t type)
{
	union8_t t8;
	union16_t t16;
	union32_t t32;
	union64_t t64;
	int ret = 0;

	switch (type) {
#define CASE(type, pyconvert, pytype, disktype, htole, bytes)	     \
		case LAYOUT_TYPE_##type:			     \
			pytype = pyconvert(val);		     \
			if (PyErr_Occurred())			     \
				return;				     \
			disktype = htole(disktype);		     \
			ret = fwrite(&disktype, bytes, 1, out);	     \
			break
		CASE(INT8,   PyInt_AsLong, t8.i,  t8.u,         , 1);
		CASE(UINT8,  PyInt_AsLong, t8.u,  t8.u,         , 1);
		CASE(INT16,  PyInt_AsLong, t16.i, t16.u, htole16, 2);
		CASE(UINT16, PyInt_AsLong, t16.u, t16.u, htole16, 2);
		CASE(INT32,  PyInt_AsLong, t32.i, t32.u, htole32, 4);
		CASE(UINT32, PyInt_AsLong, t32.u, t32.u, htole32, 4);
		CASE(INT64,  PyInt_AsLong, t64.i, t64.u, htole64, 8);
		CASE(UINT64, PyInt_AsLong, t64.u, t64.u, htole64, 8);
		CASE(FLOAT32, PyFloat_AsDouble, t32.f, t32.u, htole32, 4);
		CASE(FLOAT64, PyFloat_AsDouble, t64.d, t64.u, htole64, 8);
#undef CASE
	default:
		PyErr_SetString(PyExc_TypeError, "unknown type");
		return;
	}
	if (ret <= 0) {
		PyErr_SetFromErrno(PyExc_OSError);
	}
}
/* .append_iter(maxrows, dataiter) */
static PyObject *Rocket_append_iter(Rocket *self, PyObject *args)
{
	int maxrows;
	PyObject *iter;
	PyObject *rowlist;
	if (!PyArg_ParseTuple(args, "iO:append_iter", &maxrows, &iter))
		return NULL;
	if (!PyIter_Check(iter)) {
		PyErr_SetString(PyExc_TypeError, "need an iterable");
		return NULL;
	}
	if (!self->file) {
		PyErr_SetString(PyExc_Exception, "no file");
		return NULL;
	}

	/* Mark file size so that it will get updated next time it's read */
	self->file_size = -1;

	int row;
	for (row = 0; row < maxrows; row++) {
		rowlist = PyIter_Next(iter);
		if (!rowlist)
			break;
		if (!PyList_Check(rowlist)) {
			PyErr_SetString(PyExc_TypeError, "rows must be lists");
			goto row_err;
		}
		if (PyList_Size(rowlist) != self->layout_count + 1) {
			PyErr_SetString(PyExc_TypeError, "short row");
			goto row_err;
		}

		/* Extract and write timestamp */
		append_pyobject(self->file, PyList_GetItem(rowlist, 0),
				LAYOUT_TYPE_INT64);
		if (PyErr_Occurred())
			goto row_err;

		/* Extract and write values */
		int i;
		for (i = 0; i < self->layout_count; i++) {
			append_pyobject(self->file,
					PyList_GetItem(rowlist, i+1),
					self->layout_type);
			if (PyErr_Occurred())
				goto row_err;
		}
	}
	fflush(self->file);
	/* All done */
	return PyLong_FromLong(row);
row_err:
	fflush(self->file);
	Py_DECREF(rowlist);
	return NULL;
}

/****
 * Append from string
 */
static inline long int strtol10(const char *nptr, char **endptr) {
	return strtol(nptr, endptr, 10);
}
static inline long int strtoul10(const char *nptr, char **endptr) {
	return strtoul(nptr, endptr, 10);
}

/* .append_string(count, data, offset, linenum, start, end, last_timestamp) */
static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
{
	int count;
	const char *data;
	int offset;
	const char *linestart;
	int linenum;
	timestamp_t start;
	timestamp_t end;
	timestamp_t last_timestamp;

	int written = 0;
	char *endptr;
	union8_t t8;
	union16_t t16;
	union32_t t32;
	union64_t t64;
	int i;

	/* It would be nice to use 't#' instead of 's' for data,
	   but we need the null termination for strto*.  If we had
	   strnto* that took a length, we could use t# and not require
	   a copy. */
	if (!PyArg_ParseTuple(args, "isiiLLL:append_string", &count,
			      &data, &offset, &linenum,
			      &start, &end, &last_timestamp))
		return NULL;

	/* Skip spaces, but don't skip over a newline. */
#define SKIP_BLANK(buf) do {			\
	while (isspace(*buf)) {			\
		if (*buf == '\n')		\
			break;			\
		buf++;				\
	} } while(0)

	const char *buf = &data[offset];
	while (written < count && *buf)
	{
		linestart = buf;
		linenum++;

		/* Skip leading whitespace and commented lines */
		SKIP_BLANK(buf);
		if (*buf == '#') {
			while (*buf && *buf != '\n')
				buf++;
			if (*buf)
				buf++;
			continue;
		}

		/* Extract timestamp */
		t64.i = strtoll(buf, &endptr, 10);
		if (endptr == buf || !isspace(*endptr)) {
			/* Try parsing as a double instead */
			t64.d = strtod(buf, &endptr);
			if (endptr == buf)
				goto bad_timestamp;
			if (!isspace(*endptr))
				goto cant_parse_value;
			t64.i = round(t64.d);
		}
		if (t64.i <= last_timestamp)
			return raise_int(linenum, buf - linestart + 1,
					 ERR_NON_MONOTONIC, t64.i);
		last_timestamp = t64.i;
		if (t64.i < start || t64.i >= end)
			return raise_int(linenum, buf - linestart + 1,
					 ERR_OUT_OF_INTERVAL, t64.i);
		t64.u = le64toh(t64.u);
		if (fwrite(&t64.u, 8, 1, self->file) != 1)
			goto err;
		buf = endptr;

		/* Parse all values in the line */
		switch (self->layout_type) {
#define CS(type, parsefunc, parsetype, realtype, disktype, letoh, bytes) \
		case LAYOUT_TYPE_##type:				\
			/* parse and write in a loop */			\
			for (i = 0; i < self->layout_count; i++) {	\
				/* skip non-newlines */			\
				SKIP_BLANK(buf);			\
				if (*buf == '\n')			\
					goto wrong_number_of_values;	\
				/* parse number */			\
				parsetype = parsefunc(buf, &endptr);	\
				if (*endptr && !isspace(*endptr))	\
					goto cant_parse_value;		\
				/* check limits */			\
				if (type##_MIN != type##_MAX &&		\
				    (parsetype < type##_MIN ||		\
				     parsetype > type##_MAX))		\
					goto value_out_of_range;	\
				/* convert to disk representation */	\
				realtype = parsetype;			\
				disktype = letoh(disktype);		\
				/* write it */				\
				if (fwrite(&disktype, bytes,		\
					   1, self->file) != 1)		\
					goto err;			\
				/* advance buf */			\
				buf = endptr;				\
			}						\
			/* Skip trailing whitespace and comments */	\
			SKIP_BLANK(buf);				\
			if (*buf == '#')				\
				while (*buf && *buf != '\n')		\
					buf++;				\
			if (*buf == '\n')				\
				buf++;					\
			else if (*buf != '\0')				\
				goto extra_data_on_line;		\
			break

			CS(INT8,   strtol10,  t64.i, t8.i,  t8.u,         , 1);
			CS(UINT8,  strtoul10, t64.u, t8.u,  t8.u,         , 1);
			CS(INT16,  strtol10,  t64.i, t16.i, t16.u, le16toh, 2);
			CS(UINT16, strtoul10, t64.u, t16.u, t16.u, le16toh, 2);
			CS(INT32,  strtol10,  t64.i, t32.i, t32.u, le32toh, 4);
			CS(UINT32, strtoul10, t64.u, t32.u, t32.u, le32toh, 4);
			CS(INT64,  strtol10,  t64.i, t64.i, t64.u, le64toh, 8);
			CS(UINT64, strtoul10, t64.u, t64.u, t64.u, le64toh, 8);
			CS(FLOAT32, strtod,   t64.d, t32.f, t32.u, le32toh, 4);
			CS(FLOAT64, strtod,   t64.d, t64.d, t64.u, le64toh, 8);
#undef CS
		default:
			PyErr_SetString(PyExc_TypeError, "unknown type");
			return NULL;
		}

		/* Done this line */
		written++;
	}

	fflush(self->file);

	/* Build return value and return*/
	offset = buf - data;
	PyObject *o;
	o = Py_BuildValue("(iiLi)", written, offset, last_timestamp, linenum);
	return o;
err:
	PyErr_SetFromErrno(PyExc_OSError);
	return NULL;
bad_timestamp:
	return raise_str(linenum, buf - linestart + 1,
			 ERR_OTHER, "bad timestamp");
cant_parse_value:
	return raise_str(linenum, buf - linestart + 1,
			 ERR_OTHER, "can't parse value");
wrong_number_of_values:
	return raise_str(linenum, buf - linestart + 1,
			 ERR_OTHER, "wrong number of values");
value_out_of_range:
	return raise_str(linenum, buf - linestart + 1,
			 ERR_OTHER, "value out of range");
extra_data_on_line:
	return raise_str(linenum, buf - linestart + 1,
			 ERR_OTHER, "extra data on line");
}

/****
 * Extract to Python list
 */

static int _extract_handle_params(Rocket *self, PyObject *args, long *count)
{
	long offset;
	if (!PyArg_ParseTuple(args, "ll", &offset, count))
		return -1;
	if (!self->file) {
		PyErr_SetString(PyExc_Exception, "no file");
		return -1;
	}
	/* Seek to target location */
	if (fseek(self->file, offset, SEEK_SET) < 0) {
		PyErr_SetFromErrno(PyExc_OSError);
		return -1;
	}
	return 0;
}

/* Helper for extracting data from a file as a Python object */
static inline void *extract_pyobject(FILE *in, layout_type_t type)
{
	union8_t t8;
	union16_t t16;
	union32_t t32;
	union64_t t64;

	switch (type) {
#define CASE(type, pyconvert, pytype, disktype, letoh, bytes)		\
		case LAYOUT_TYPE_##type:				\
			if (fread(&disktype, bytes, 1, in) <= 0)	\
				break;					\
			disktype = letoh(disktype);			\
			return pyconvert(pytype);			\
			break
		CASE(INT8,   PyInt_FromLong, t8.i,  t8.u,         , 1);
		CASE(UINT8,  PyInt_FromLong, t8.u,  t8.u,         , 1);
		CASE(INT16,  PyInt_FromLong, t16.i, t16.u, le16toh, 2);
		CASE(UINT16, PyInt_FromLong, t16.u, t16.u, le16toh, 2);
		CASE(INT32,  PyInt_FromLong, t32.i, t32.u, le32toh, 4);
		CASE(UINT32, PyInt_FromLong, t32.u, t32.u, le32toh, 4);
		CASE(INT64,  PyInt_FromLong, t64.i, t64.u, le64toh, 8);
		CASE(UINT64, PyInt_FromLong, t64.u, t64.u, le64toh, 8);
		CASE(FLOAT32, PyFloat_FromDouble, t32.f, t32.u, le32toh, 4);
		CASE(FLOAT64, PyFloat_FromDouble, t64.d, t64.u, le64toh, 8);
#undef CASE
	default:
		PyErr_SetString(PyExc_TypeError, "unknown type");
		return NULL;
	}
	PyErr_SetString(PyExc_OSError, "failed to read from file");
	return NULL;
}

static PyObject *Rocket_extract_list(Rocket *self, PyObject *args)
{
	long count;
	if (_extract_handle_params(self, args, &count) < 0)
		return NULL;

	/* Make a list to return */
	PyObject *retlist = PyList_New(0);
	if (!retlist)
		return NULL;

	/* Read data into new Python lists */
	int row;
	for (row = 0; row < count; row++)
	{
		PyObject *rowlist = PyList_New(self->layout_count + 1);
		if (!rowlist) {
			Py_DECREF(retlist);
			return NULL;
		}

		/* Timestamp */
		PyObject *entry = extract_pyobject(self->file,
						   LAYOUT_TYPE_INT64);
		if (!entry || (PyList_SetItem(rowlist, 0, entry) < 0)) {
			Py_DECREF(rowlist);
			Py_DECREF(retlist);
			return NULL;
		}

		/* Data */
		int i;
		for (i = 0; i < self->layout_count; i++) {
			PyObject *ent = extract_pyobject(self->file,
							 self->layout_type);
			if (!ent || (PyList_SetItem(rowlist, i+1, ent) < 0)) {
				Py_DECREF(rowlist);
				Py_DECREF(retlist);
				return NULL;
			}
		}

		/* Add row to return value */
		if (PyList_Append(retlist, rowlist) < 0) {
			Py_DECREF(rowlist);
			Py_DECREF(retlist);
			return NULL;
		}

		Py_DECREF(rowlist);
	}
	return retlist;
}

/****
 * Extract to string
 */

static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
{
	long count;
	if (_extract_handle_params(self, args, &count) < 0)
		return NULL;

	char *str = NULL, *new;
	long len_alloc = 0;
	long len = 0;
	int ret;

	/* min space free in string (and the maximum length of one
	   line); this is generous */
	const int min_free = 32 * MAX_LAYOUT_COUNT;

	/* how much to allocate at once */
	const int alloc_size = 1048576;

	int row, i;
	union8_t t8;
	union16_t t16;
	union32_t t32;
	union64_t t64;
	for (row = 0; row < count; row++) {
		/* Make sure there's space for a line */
		if ((len_alloc - len) < min_free) {
			/* grow by 1 meg at a time */
			len_alloc += alloc_size;
			new = realloc(str, len_alloc);
			if (new == NULL)
				goto err;
			str = new;
		}

		/* Read and print timestamp */
		if (fread(&t64.u, 8, 1, self->file) != 1)
			goto err;
		t64.u = le64toh(t64.u);
		ret = sprintf(&str[len], "%ld", t64.i);
		if (ret <= 0)
			goto err;
		len += ret;

		/* Read and print values */
		switch (self->layout_type) {
#define CASE(type, fmt, fmttype, disktype, letoh, bytes)		\
		case LAYOUT_TYPE_##type:				\
			/* read and format in a loop */			\
			for (i = 0; i < self->layout_count; i++) {	\
				if (fread(&disktype, bytes,		\
					  1, self->file) < 0)		\
					goto err;			\
				disktype = letoh(disktype);		\
				ret = sprintf(&str[len], " " fmt,	\
					      fmttype);			\
				if (ret <= 0)				\
					goto err;			\
				len += ret;				\
			}						\
			break
			CASE(INT8,   "%hhd",   t8.i,  t8.u,         , 1);
			CASE(UINT8,  "%hhu",   t8.u,  t8.u,         , 1);
			CASE(INT16,  "%hd",    t16.i, t16.u, le16toh, 2);
			CASE(UINT16, "%hu",    t16.u, t16.u, le16toh, 2);
			CASE(INT32,  "%d",     t32.i, t32.u, le32toh, 4);
			CASE(UINT32, "%u",     t32.u, t32.u, le32toh, 4);
			CASE(INT64,  "%ld",    t64.i, t64.u, le64toh, 8);
			CASE(UINT64, "%lu",    t64.u, t64.u, le64toh, 8);
			/* These next two are a bit debatable.  floats
			   are 6-9 significant figures, so we print 7.
			   Doubles are 15-19, so we print 17.  This is
			   similar to the old prep format for float32.
			*/
			CASE(FLOAT32, "%.6e",  t32.f, t32.u, le32toh, 4);
			CASE(FLOAT64, "%.16e", t64.d, t64.u, le64toh, 8);
#undef CASE
		default:
			PyErr_SetString(PyExc_TypeError, "unknown type");
			if (str) free(str);
			return NULL;
		}
		str[len++] = '\n';
	}

	PyObject *pystr = PyString_FromStringAndSize(str, len);
	free(str);
	return pystr;
err:
	if (str) free(str);
	PyErr_SetFromErrno(PyExc_OSError);
	return NULL;
}

/****
 * Module and type setup
 */

static PyGetSetDef Rocket_getsetters[] = {
	{ "file_size", (getter)Rocket_get_file_size, NULL,
	  "file size in bytes", NULL },
	{ NULL },
};

static PyMemberDef Rocket_members[] = {
	{ "binary_size", T_INT, offsetof(Rocket, binary_size), 0,
	  "binary size per row" },
	{ NULL },
};

static PyMethodDef Rocket_methods[] = {
	{ "close", (PyCFunction)Rocket_close, METH_NOARGS,
	  "close(self)\n\n"
	  "Close file handle" },

	{ "append_iter", (PyCFunction)Rocket_append_iter, METH_VARARGS,
	  "append_iter(self, maxrows, iterable)\n\n"
	  "Append up to maxrows of data from iter to the file" },

	{ "append_string", (PyCFunction)Rocket_append_string, METH_VARARGS,
	  "append_string(self, count, data, offset, line, start, end, ts)\n\n"
          "Parse string and append data.\n"
	  "\n"
	  "  count: maximum number of rows to add\n"
          "  data: string data\n"
          "  offset: byte offset into data to start parsing\n"
          "  line: current line number of data\n"
          "  start: starting timestamp for interval\n"
          "  end: end timestamp for interval\n"
          "  ts: last timestamp that was previously parsed\n"
	  "\n"
	  "Raises ParseError if timestamps are non-monotonic, outside\n"
	  "the start/end interval etc.\n"
	  "\n"
          "On success, return a tuple with three values:\n"
          "  added_rows: how many rows were added from the file\n"
          "  data_offset: current offset into the data string\n"
          "  last_timestamp: last timestamp we parsed" },

	{ "extract_list", (PyCFunction)Rocket_extract_list, METH_VARARGS,
	  "extract_list(self, offset, count)\n\n"
	  "Extract count rows of data from the file at offset offset.\n"
	  "Return a list of lists [[row],[row],...]" },

	{ "extract_string", (PyCFunction)Rocket_extract_string, METH_VARARGS,
	  "extract_string(self, offset, count)\n\n"
	  "Extract count rows of data from the file at offset offset.\n"
	  "Return an ascii formatted string according to the layout" },

	{ NULL },
};

static PyTypeObject RocketType = {
	PyObject_HEAD_INIT(NULL)

	.tp_name	= "rocket.Rocket",
	.tp_basicsize	= sizeof(Rocket),
	.tp_flags	= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,

	.tp_new		= Rocket_new,
	.tp_dealloc	= (destructor)Rocket_dealloc,
	.tp_init	= (initproc)Rocket_init,
	.tp_methods	= Rocket_methods,
	.tp_members	= Rocket_members,
	.tp_getset	= Rocket_getsetters,

	.tp_doc		= ("rocket.Rocket(layout, file)\n\n"
			   "C implementation of the \"rocket\" data parsing\n"
			   "interface, which translates between the binary\n"
			   "format on disk and the ASCII or Python list\n"
			   "format used when communicating with the rest of\n"
			   "the system.")
};

static PyMethodDef module_methods[] = {
	{ NULL },
};

PyMODINIT_FUNC
initrocket(void)
{
	PyObject *module;

	RocketType.tp_new = PyType_GenericNew;
	if (PyType_Ready(&RocketType) < 0)
		return;

	module = Py_InitModule3("rocket", module_methods,
				"Rocket data parsing and formatting module");
	Py_INCREF(&RocketType);
	PyModule_AddObject(module, "Rocket", (PyObject *)&RocketType);

	ParseError = PyErr_NewException("rocket.ParseError", NULL, NULL);
	Py_INCREF(ParseError);
	PyModule_AddObject(module, "ParseError", ParseError);
	add_parseerror_codes(module);

	return;
}