Browse Source

Add rocket.extract_list; still not as complete as pyrocket

tags/nilmdb-1.3.0
Jim Paris 11 years ago
parent
commit
7c95934cc2
2 changed files with 153 additions and 25 deletions
  1. +2
    -2
      nilmdb/server/bulkdata.py
  2. +151
    -23
      nilmdb/server/rocket.c

+ 2
- 2
nilmdb/server/bulkdata.py View File

@@ -341,14 +341,14 @@ class Table(object):
depending on as_string."""
if (start is None or
stop is None or
start >= stop or
start < 0 or
stop > self.nrows):
raise IndexError("Index out of range")

ret = []
row = start
remaining = stop - start
while remaining:
while remaining > 0:
(subdir, filename, offset, count) = self._offset_from_row(row)
if count > remaining:
count = remaining


+ 151
- 23
nilmdb/server/rocket.c View File

@@ -13,16 +13,6 @@ typedef enum {
LAYOUT_TYPE_FLOAT64,
} layout_type_t;

const int layout_size[] = {
[LAYOUT_TYPE_NONE] = 0,
[LAYOUT_TYPE_UINT8] = 1,
[LAYOUT_TYPE_UINT16] = 2,
[LAYOUT_TYPE_UINT32] = 4,
[LAYOUT_TYPE_UINT64] = 8,
[LAYOUT_TYPE_FLOAT32] = 4,
[LAYOUT_TYPE_FLOAT64] = 8
};

struct {
char *string;
layout_type_t layout;
@@ -121,11 +111,27 @@ FILE *PyFile_AsFile(PyObject *file)
/* Convert numbers, being careful with aliasing rules */
static inline uint32_t float_to_uint32(float x)
{
return *(uint32_t *)(char *)&x;
union { float f; uint32_t u; } u;
u.f = x;
return u.u;
}
static inline uint64_t double_to_uint64(double x)
{
return *(uint64_t *)(char *)&x;
union { double f; uint64_t u; } u;
u.f = x;
return u.u;
}
static inline float uint32_to_float(uint32_t x)
{
union { float f; uint32_t u; } u;
u.u = x;
return u.f;
}
static inline double uint64_to_double(uint64_t x)
{
union { double f; uint64_t u; } u;
u.u = x;
return u.f;
}

static inline void write_binary(FILE *out, uint64_t val, layout_type_t type)
@@ -158,7 +164,7 @@ static inline void write_binary(FILE *out, uint64_t val, layout_type_t type)
PyErr_SetString(PyExc_TypeError, "unknown type");
return;
}
if (ret == 0)
if (ret != 1)
PyErr_SetFromErrno(PyExc_OSError);
}

@@ -182,6 +188,7 @@ static inline void write_pyobject(FILE *out, PyObject *val, layout_type_t type)
if (PyErr_Occurred())
return;
write_binary(out, float_to_uint32(d), type);
break;
case LAYOUT_TYPE_FLOAT64:
d = PyFloat_AsDouble(val);
if (PyErr_Occurred())
@@ -194,6 +201,69 @@ static inline void write_pyobject(FILE *out, PyObject *val, layout_type_t type)
}
}

static inline uint64_t read_binary(FILE *in, layout_type_t type)
{
int ret = 0;
uint8_t x8;
uint16_t x16;
uint32_t x32;
uint64_t x64;
switch (type) {
case LAYOUT_TYPE_UINT8:
ret = fread(&x8, 1, 1, in);
x64 = x8;
break;
case LAYOUT_TYPE_UINT16:
ret = fread(&x16, 2, 1, in);
x64 = le16toh(x16);
break;
case LAYOUT_TYPE_UINT32:
case LAYOUT_TYPE_FLOAT32:
ret = fread(&x32, 4, 1, in);
x64 = le32toh(x32);
break;
case LAYOUT_TYPE_UINT64:
case LAYOUT_TYPE_FLOAT64:
ret = fread(&x64, 8, 1, in);
x64 = le64toh(x64);
break;
default:
PyErr_SetString(PyExc_TypeError, "unknown type");
return 0;
}
if (ret == 0) {
if (feof(in))
PyErr_SetString(PyExc_OSError, "unexpected EOF");
else
PyErr_SetFromErrno(PyExc_OSError);
return 0;
}
return x64;
}

static inline void *read_pyobject(FILE *in, layout_type_t type)
{
uint64_t val;

val = read_binary(in, type);
if (!val && PyErr_Occurred())
return NULL;
switch (type) {
case LAYOUT_TYPE_UINT8:
case LAYOUT_TYPE_UINT16:
case LAYOUT_TYPE_UINT32:
case LAYOUT_TYPE_UINT64:
return PyInt_FromLong(val);
case LAYOUT_TYPE_FLOAT32:
return PyFloat_FromDouble(uint32_to_float(val));
case LAYOUT_TYPE_FLOAT64:
return PyFloat_FromDouble(uint64_to_double(val));
default:
PyErr_SetString(PyExc_TypeError, "unknown type");
return NULL;
}
}

static PyObject *Rocket_append_list(Rocket *self, PyObject *args)
{
PyObject *file, *list;
@@ -213,29 +283,35 @@ static PyObject *Rocket_append_list(Rocket *self, PyObject *args)
PyObject *rowlist = PyList_GetItem(list, row);
if (!PyList_Check(list)) {
PyErr_SetString(PyExc_TypeError, "rows must be lists");
fflush(out);
return NULL;
}
if (PyList_Size(rowlist) != self->layout_count + 1) {
PyErr_SetString(PyExc_TypeError, "short row list");
fflush(out);
return NULL;
}

/* Extract and write timestamp */
write_pyobject(out, PyList_GetItem(rowlist, 0),
LAYOUT_TYPE_FLOAT64);
if (PyErr_Occurred())
if (PyErr_Occurred()) {
fflush(out);
return NULL;
}

/* Extract and write values */
int i;
for (i = 0; i < self->layout_count; i++) {
write_pyobject(out, PyList_GetItem(rowlist, i+1),
self->layout_type);
if (PyErr_Occurred())
if (PyErr_Occurred()) {
fflush(out);
return NULL;
}
}
}
fflush(out);
/* All done */
Py_INCREF(Py_None);
return Py_None;
@@ -243,18 +319,70 @@ static PyObject *Rocket_append_list(Rocket *self, PyObject *args)

static PyObject *Rocket_extract_list(Rocket *self, PyObject *args)
{
PyObject *file, *offset, *count;
PyObject *pyfile, *pyoffset, *pycount;
FILE *in;
if (!PyArg_ParseTuple(args, "OOO:extract_list",
&file, &offset, &count))
&pyfile, &pyoffset, &pycount))
return NULL;
if ((in = PyFile_AsFile(file)) == NULL)
if ((in = PyFile_AsFile(pyfile)) == NULL)
return NULL;
long offset = PyLong_AsLong(pyoffset);
if (PyErr_Occurred())
return NULL;
long count = PyLong_AsLong(pycount);
if (PyErr_Occurred())
return NULL;

PyErr_SetString(PyExc_NotImplementedError, "uh oh");
return NULL;
Py_INCREF(Py_None);
return Py_None;
/* Seek to target location */
if (fseek(in, offset, SEEK_SET) < 0) {
PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}

/* Make a list to return */
PyObject *retlist = PyList_New(0);
if (!retlist)
return NULL;

/* Read data into new Python lists */
int row;
for (row = 0; row < count; row++)
{
PyObject *rowlist = PyList_New(self->layout_count + 1);
if (!rowlist) {
Py_DECREF(retlist);
return NULL;
}

/* Timestamp */
PyObject *entry = read_pyobject(in, LAYOUT_TYPE_FLOAT64);
if (!entry || (PyList_SetItem(rowlist, 0, entry) < 0)) {
Py_DECREF(rowlist);
Py_DECREF(retlist);
return NULL;
}

/* Data */
int i;
for (i = 0; i < self->layout_count; i++) {
PyObject *ent = read_pyobject(in, self->layout_type);
if (!ent || (PyList_SetItem(rowlist, i+1, ent) < 0)) {
Py_DECREF(rowlist);
Py_DECREF(retlist);
return NULL;
}
}

/* Add row to return value */
if (PyList_Append(retlist, rowlist) < 0) {
Py_DECREF(rowlist);
Py_DECREF(retlist);
return NULL;
}

Py_DECREF(rowlist);
}
return retlist;
}

static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)


Loading…
Cancel
Save