Fill out client tests and fix various bugs

Fixes various corner cases and other bugs regarding lines with
comments, having data but no endpoints, etc.

nilmdb/client/client.py

@@ -5,6 +5,7 @@
 import nilmdb
 import nilmdb.utils
 import nilmdb.client.httpclient
+from nilmdb.client.errors import ClientError
 
 import time
 import simplejson as json
@@ -386,10 +387,12 @@ class StreamInserter(object):
 
         # Double check endpoints
         if start_ts is None or end_ts is None:
-            if len(block) == 0:
-                # No data is OK
+            # If the block has no non-comment lines, it's OK
+            try:
+                self._get_first_noncomment(block)
+            except IndexError:
                 return
-            raise ValueError("have data to send, but no start/end times")
+            raise ClientError("have data to send, but no start/end times")
 
         # Send it
         params = { "path": self._path,

nilmdb/server/rocket.c

@@ -379,6 +379,17 @@ static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
 	{
 		linenum++;
 
+		/* Skip leading whitespace and commented lines */
+		while (*buf == ' ' || *buf == '\t')
+			buf++;
+		if (*buf == '#') {
+			while (*buf && *buf != '\n')
+				buf++;
+			if (*buf)
+				buf++;
+			continue;
+		}
+
 		/* Extract timestamp */
 		t64.d = strtod(buf, &endptr);
 		if (endptr == buf)
@@ -413,8 +424,12 @@ static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
 					goto err;			\
 				buf = endptr;				\
 			}						\
+			/* Skip trailing whitespace and comments */	\
 			while (*buf == ' ' || *buf == '\t')		\
 				buf++;					\
+			if (*buf == '#')				\
+				while (*buf && *buf != '\n')		\
+					buf++;				\
 			if (*buf == '\n')				\
 				buf++;					\
 			else if (*buf != '\0')				\

nilmdb/utils/time.py

@@ -60,8 +60,6 @@ def format_time(timestamp):
     Convert a Unix timestamp to a string for printing, using the
     local timezone for display (e.g. from the TZ env var).
     """
-    if "totimestamp" in dir(timestamp):
-        timestamp = timestamp.totimestamp()
     dt = datetime_tz.datetime_tz.fromtimestamp(timestamp)
     return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")
 

nilmdb/utils/timestamper.py

@@ -70,9 +70,6 @@ class TimestamperRate(Timestamper):
                     raise StopIteration
                 yield sprintf("%.6f ", start + n / rate)
                 n += 1
-        # Handle case where we're passed a datetime or datetime_tz object
-        if "totimestamp" in dir(start):
-            start = start.totimestamp()
         Timestamper.__init__(self, infile, iterator(start, rate, end))
         self.start = start
         self.rate = rate

tests/test_client.py

@@ -448,7 +448,9 @@ class TestClient(object):
             ctx.insert("\n103 1\n")
 
             ctx.insert("104 1\n")
-            ctx.insert("105 1\n")
+            ctx.insert("# hello\n")
+            ctx.insert("   # hello\n")
+            ctx.insert("  105 1\n")
             ctx.finalize()
 
             ctx.insert("106 1\n")
@@ -465,8 +467,11 @@ class TestClient(object):
             ctx.update_end(114)
             ctx.insert("113 1\n")
             ctx.update_end(115)
-            ctx.insert("114 1\n")
+            ctx.insert("114 1" +
+                       " # this is super long" * 100 +
+                       "\n")
             ctx.finalize()
+            ctx.insert("# this is super long" * 100)
 
         with assert_raises(ClientError):
             with client.stream_insert_context("/context/test", 100, 200) as ctx:
@@ -476,6 +481,10 @@ class TestClient(object):
             with client.stream_insert_context("/context/test", 200, 300) as ctx:
                 ctx.insert("115 1\n")
 
+        with assert_raises(ClientError):
+            with client.stream_insert_context("/context/test") as ctx:
+                ctx.insert("bogus data\n")
+
         with client.stream_insert_context("/context/test", 200, 300) as ctx:
             # make sure our override wasn't permanent
             ne_(ctx._max_data, 15)
@@ -566,8 +575,15 @@ class TestClient(object):
             ctx.update_end(1350)
             ctx.finalize() # nothing
             ctx.update_start(1400)
+            ctx.insert("# nothing!\n")
             ctx.update_end(1450)
             ctx.finalize()
+            ctx.update_start(1500)
+            ctx.insert("# nothing!")
+            ctx.update_end(1550)
+            ctx.finalize()
+            ctx.insert("# nothing!\n" * 10)
+            ctx.finalize()
             # implicit last finalize inserts [1400, 1450]
 
         # Check everything
@@ -580,6 +596,7 @@ class TestClient(object):
                      (1, [1100, 1100.000001]),
                      (1, [1199, 1250]),
                      (0, [1400, 1450]),
+                     (0, [1500, 1550]),
                      ])
 
         # Clean up