Browse Source

Convert times to microsecond precision strings more consistently.

Use a new helper, nilmdb.utils.time.float_to_time_string().
This will help if we ever want to change representation (like using
uint64 microseconds since epoch, which saves us from having to
waste bits on the floating-point exponent)
tags/nilmdb-1.3.0
Jim Paris 9 years ago
parent
commit
11b228f77a
9 changed files with 30 additions and 23 deletions
  1. +9
    -11
      nilmdb/client/client.py
  2. +1
    -1
      nilmdb/cmdline/extract.py
  3. +1
    -1
      nilmdb/cmdline/list.py
  4. +3
    -1
      nilmdb/server/interval.pyx
  5. +1
    -0
      nilmdb/utils/__init__.py
  6. +5
    -0
      nilmdb/utils/time.py
  7. +2
    -2
      tests/data/extract-7
  8. +3
    -3
      tests/test_cmdline.py
  9. +5
    -4
      tests/test_interval.py

+ 9
- 11
nilmdb/client/client.py View File

@@ -10,9 +10,7 @@ import time
import simplejson as json
import contextlib

def float_to_string(f):
"""Use repr to maintain full precision in the string output."""
return repr(float(f))
from nilmdb.utils.time import float_time_to_string

def extract_timestamp(line):
"""Extract just the timestamp from a line of data text"""
@@ -102,9 +100,9 @@ class Client(object):
"path": path
}
if start is not None:
params["start"] = float_to_string(start)
params["start"] = float_time_to_string(start)
if end is not None:
params["end"] = float_to_string(end)
params["end"] = float_time_to_string(end)
return self.http.post("stream/remove", params)

@contextlib.contextmanager
@@ -143,8 +141,8 @@ class Client(object):
stream_insert, except 'block' contains multiple lines of ASCII
text and is sent in one single chunk."""
params = { "path": path,
"start": float_to_string(start),
"end": float_to_string(end) }
"start": float_time_to_string(start),
"end": float_time_to_string(end) }
return self.http.put("stream/insert", block, params)

def stream_intervals(self, path, start = None, end = None):
@@ -155,9 +153,9 @@ class Client(object):
"path": path
}
if start is not None:
params["start"] = float_to_string(start)
params["start"] = float_time_to_string(start)
if end is not None:
params["end"] = float_to_string(end)
params["end"] = float_time_to_string(end)
return self.http.get_gen("stream/intervals", params)

def stream_extract(self, path, start = None, end = None, count = False):
@@ -173,9 +171,9 @@ class Client(object):
"path": path,
}
if start is not None:
params["start"] = float_to_string(start)
params["start"] = float_time_to_string(start)
if end is not None:
params["end"] = float_to_string(end)
params["end"] = float_time_to_string(end)
if count:
params["count"] = 1
return self.http.get_gen("stream/extract", params)


+ 1
- 1
nilmdb/cmdline/extract.py View File

@@ -44,7 +44,7 @@ def cmd_extract(self):
layout = streams[0][1]

if self.args.timestamp_raw:
time_string = repr
time_string = nilmdb.utils.time.float_time_to_string
else:
time_string = nilmdb.utils.time.format_time



+ 1
- 1
nilmdb/cmdline/list.py View File

@@ -68,7 +68,7 @@ def cmd_list(self):
streams = self.client.stream_list(extent = True)

if self.args.timestamp_raw:
time_string = repr
time_string = nilmdb.utils.time.float_time_to_string
else:
time_string = nilmdb.utils.time.format_time



+ 3
- 1
nilmdb/server/interval.pyx View File

@@ -19,6 +19,8 @@ Intervals are half-open, ie. they include data points with timestamps
# Fourth version is an optimized rb-tree that stores interval starts
# and ends directly in the tree, like bxinterval did.

from ..utils.time import float_time_to_string as ftts

cimport rbtree
cdef extern from "stdint.h":
ctypedef unsigned long long uint64_t
@@ -47,7 +49,7 @@ cdef class Interval:
return self.__class__.__name__ + "(" + s + ")"

def __str__(self):
return "[" + repr(self.start) + " -> " + repr(self.end) + ")"
return "[" + ftts(self.start) + " -> " + ftts(self.end) + ")"

def __cmp__(self, Interval other):
"""Compare two intervals. If non-equal, order by start then end"""


+ 1
- 0
nilmdb/utils/__init__.py View File

@@ -8,3 +8,4 @@ from nilmdb.utils.diskusage import du, human_size
from nilmdb.utils.mustclose import must_close
from nilmdb.utils import atomic
import nilmdb.utils.threadsafety
import nilmdb.utils.time

+ 5
- 0
nilmdb/utils/time.py View File

@@ -52,3 +52,8 @@ def format_time(timestamp):
"""
dt = datetime_tz.datetime_tz.fromtimestamp(timestamp)
return dt.strftime("%a, %d %b %Y %H:%M:%S.%f %z")

def float_time_to_string(timestamp):
"""Convert a floating-point Unix timestamp to a string,
like '1234567890.000000'"""
return "%.6f" % timestamp

+ 2
- 2
tests/data/extract-7 View File

@@ -1,7 +1,7 @@
# path: /newton/prep
# layout: float32_8
# start: 1332496830.0
# end: 1332496830.999
# start: 1332496830.000000
# end: 1332496830.999000
1332496830.000000 2.517740e+05 2.242410e+05 5.688100e+03 1.915530e+03 9.329220e+03 4.183710e+03 1.212350e+03 2.641790e+03
1332496830.008333 2.595670e+05 2.226980e+05 6.207600e+03 6.786720e+02 9.380230e+03 4.575580e+03 2.830610e+03 2.688630e+03
1332496830.016667 2.630730e+05 2.233040e+05 4.961640e+03 2.197120e+03 7.687310e+03 4.861860e+03 2.732780e+03 3.008540e+03


+ 3
- 3
tests/test_cmdline.py View File

@@ -480,19 +480,19 @@ class TestCmdline(object):
self.ok("list --detail --path *prep --timestamp-raw "
"--start='23 Mar 2012 10:05:15.50'")
lines_(self.captured, 2)
self.contain("[ 1332497115.5 -> 1332497159.991668 ]")
self.contain("[ 1332497115.500000 -> 1332497159.991668 ]")

self.ok("list --detail --path *prep -T "
"--start='23 Mar 2012 10:05:15.612'")
lines_(self.captured, 2)
self.contain("[ 1332497115.612 -> 1332497159.991668 ]")
self.contain("[ 1332497115.612000 -> 1332497159.991668 ]")

# Check --extent output
self.ok("list --extent")
lines_(self.captured, 6)

self.ok("list -E -T")
self.contain(" extent: 1332496800 -> 1332497159.991668")
self.contain(" extent: 1332496800.000000 -> 1332497159.991668")
self.contain(" extent: (no data)")

# Misc


+ 5
- 4
tests/test_interval.py View File

@@ -89,14 +89,14 @@ class TestInterval:

# big integers and floats
x = Interval(5000111222, 6000111222)
eq_(str(x), "[5000111222.0 -> 6000111222.0)")
eq_(str(x), "[5000111222.000000 -> 6000111222.000000)")
x = Interval(123.45, 234.56)
eq_(str(x), "[123.45 -> 234.56)")
eq_(str(x), "[123.450000 -> 234.560000)")

# misc
i = Interval(d1, d2)
eq_(repr(i), repr(eval(repr(i))))
eq_(str(i), "[1332561600.0 -> 1332648000.0)")
eq_(str(i), "[1332561600.000000 -> 1332648000.000000)")

def test_interval_intersect(self):
# Test Interval intersections
@@ -192,7 +192,8 @@ class TestInterval:

# misc
eq_(repr(iset), repr(eval(repr(iset))))
eq_(str(iset), "[[100.0 -> 200.0), [200.0 -> 300.0)]")
eq_(str(iset),
"[[100.000000 -> 200.000000), [200.000000 -> 300.000000)]")

def test_intervalset_geniset(self):
# Test basic iset construction


Loading…
Cancel
Save