Make rocket/bulkdata errors include column number and the bad data

The bad line is printed out on a new line, and a third line
with a ^ to indicate the position of the error.

nilmdb/server/bulkdata.py

@@ -367,22 +367,27 @@ class Table(object):
                      ) = f.append_string(count, data, data_offset, linenum,
                                          start, end, last_timestamp)
                 except rocket.ParseError as e:
-                    (linenum, errtype, obj) = e.args
+                    (linenum, colnum, errtype, obj) = e.args
+                    where = "line %d, column %d: " % (linenum, colnum)
+                    # Extract out the error line, add column marker
+                    try:
+                        bad = data.splitlines()[linenum-1]
+                        badptr = ' ' * (colnum - 1) + '^'
+                    except IndexError: # pragma: no cover
+                        bad = ""
                     if errtype == rocket.ERR_NON_MONOTONIC:
-                        err = sprintf("line %d: timestamp is not monotonically "
-                                      "increasing", linenum)
+                        err = "timestamp is not monotonically increasing"
                     elif errtype == rocket.ERR_OUT_OF_INTERVAL:
                         if obj < start:
-                            err = sprintf("line %d: Data timestamp %s < "
-                                          "start time %s", linenum,
+                            err = sprintf("Data timestamp %s < start time %s",
                                           ftts(obj), ftts(start))
                         else:
-                            err = sprintf("line %d: Data timestamp %s >= "
-                                          "end time %s", linenum,
+                            err = sprintf("Data timestamp %s >= end time %s",
                                           ftts(obj), ftts(end))
                     else:
-                        err = sprintf("line %d: %s", linenum, str(obj))
-                    raise ValueError("error parsing input data: " + err)
+                        err = str(obj)
+                    raise ValueError("error parsing input data: " +
+                                     where + err + "\n" + bad + "\n" + badptr)
                 tot_rows += added_rows
         except Exception:
             # Some failure, so try to roll things back by truncating or

nilmdb/server/pyrocket.py

@@ -117,12 +117,12 @@ class Rocket(object):
             try:
                 (ts, row) = self.layoutparser.parse(line)
             except ValueError as e:
-                raise ParseError(linenum, ERR_UNKNOWN, e)
+                raise ParseError(linenum, 0, ERR_UNKNOWN, e)
             if ts <= last_timestamp:
-                raise ParseError(linenum, ERR_NON_MONOTONIC, ts)
+                raise ParseError(linenum, 0, ERR_NON_MONOTONIC, ts)
             last_timestamp = ts
             if ts < start or ts >= end:
-                raise ParseError(linenum, ERR_OUT_OF_INTERVAL, ts)
+                raise ParseError(linenum, 0, ERR_OUT_OF_INTERVAL, ts)
             self.append_iter(1, [row])
             written += 1
         return (written, indata.tell(), last_timestamp, linenum)

nilmdb/server/rocket.c

@@ -35,20 +35,20 @@ static void add_parseerror_codes(PyObject *module)
 }
 
 /* Helpers to raise ParseErrors.  Use "return raise_str(...)" etc. */
-static PyObject *raise_str(int linenum, int code, const char *string)
+static PyObject *raise_str(int line, int col, int code, const char *string)
 {
 	PyObject *o;
-	o = Py_BuildValue("(iis)", linenum, code, string);
+	o = Py_BuildValue("(iiis)", line, col, code, string);
 	if (o != NULL) {
 		PyErr_SetObject(ParseError, o);
 		Py_DECREF(o);
 	}
 	return NULL;
 }
-static PyObject *raise_num(int linenum, int code, double num)
+static PyObject *raise_num(int line, int col, int code, double num)
 {
 	PyObject *o;
-	o = Py_BuildValue("(iid)", linenum, code, num);
+	o = Py_BuildValue("(iiid)", line, col, code, num);
 	if (o != NULL) {
 		PyErr_SetObject(ParseError, o);
 		Py_DECREF(o);
@@ -352,6 +352,7 @@ static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
 	int count;
 	const char *data;
 	int offset;
+	const char *linestart;
 	int linenum;
 	double start;
 	double end;
@@ -377,6 +378,7 @@ static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
 	const char *buf = &data[offset];
 	while (written < count && *buf)
 	{
+		linestart = buf;
 		linenum++;
 
 		/* Skip leading whitespace and commented lines */
@@ -393,12 +395,15 @@ static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
 		/* Extract timestamp */
 		t64.d = strtod(buf, &endptr);
 		if (endptr == buf)
-			return raise_str(linenum, ERR_OTHER, "bad timestamp");
+			return raise_str(linenum, buf - linestart + 1,
+					 ERR_OTHER, "bad timestamp");
 		if (t64.d <= last_timestamp)
-			return raise_num(linenum, ERR_NON_MONOTONIC, t64.d);
+			return raise_num(linenum, buf - linestart + 1,
+					 ERR_NON_MONOTONIC, t64.d);
 		last_timestamp = t64.d;
 		if (t64.d < start || t64.d >= end)
-			return raise_num(linenum, ERR_OUT_OF_INTERVAL, t64.d);
+			return raise_num(linenum, buf - linestart + 1,
+					 ERR_OUT_OF_INTERVAL, t64.d);
 		t64.u = le64toh(t64.u);
 		if (fwrite(&t64.u, 8, 1, self->file) != 1)
 			goto err;
@@ -467,11 +472,14 @@ err:
 	PyErr_SetFromErrno(PyExc_OSError);
 	return NULL;
 wrong_number_of_values:
-	return raise_str(linenum, ERR_OTHER, "wrong number of values");
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "wrong number of values");
 value_out_of_range:
-	return raise_str(linenum, ERR_OTHER, "value out of range");
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "value out of range");
 extra_data_on_line:
-	return raise_str(linenum, ERR_OTHER, "extra data on line");
+	return raise_str(linenum, buf - linestart + 1,
+			 ERR_OTHER, "extra data on line");
 }
 
 /****

tests/test_cmdline.py

@@ -394,7 +394,7 @@ class TestCmdline(object):
             self.fail("insert -s 20120323T1004 -e 20120323T1006 /newton/prep",
                       input)
             self.contain("error parsing input data")
-            self.contain("line 7:")
+            self.contain("line 7")
             self.contain("timestamp is not monotonically increasing")
 
         # insert pre-timestamped data, from stdin