From 0808ed5bd8618bdb27e25c4273679c2e3a0d44e5 Mon Sep 17 00:00:00 2001 From: Jim Paris Date: Sat, 24 Mar 2012 17:32:11 +0000 Subject: [PATCH] Add datetime_tz module from python-datetime-tz repository. This could go away if it gets packaged in Debian etc. Add nilmdb.timestamper classes Start test_timestamper (needs work) Number the functions in test_client so they run in the right order to leave the database in a good state. Fix some brokenness with module importing and namespaces git-svn-id: https://bucket.mit.edu/svn/nilm/nilmdb@10643 ddd99763-3ecb-0310-9145-efcb8ce7c51f --- datetime_tz/__init__.py | 710 ++++++++++++++++++++++++++++++++++++++ datetime_tz/pytz_abbr.py | 230 ++++++++++++ nilmdb/__init__.py | 6 + nilmdb/client.py | 15 +- nilmdb/server.py | 2 +- nilmdb/timestamper.py | 58 ++++ nilmtool.py | 1 + setup.cfg | 3 +- tests/test_client.py | 28 +- tests/test_timestamper.py | 30 ++ 10 files changed, 1058 insertions(+), 25 deletions(-) create mode 100644 datetime_tz/__init__.py create mode 100644 datetime_tz/pytz_abbr.py create mode 100644 nilmdb/timestamper.py create mode 100644 tests/test_timestamper.py diff --git a/datetime_tz/__init__.py b/datetime_tz/__init__.py new file mode 100644 index 0000000..02f2334 --- /dev/null +++ b/datetime_tz/__init__.py @@ -0,0 +1,710 @@ +#!/usr/bin/python +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Disable the invalid name warning as we are inheriting from a standard library +# object. +# pylint: disable-msg=C6409,W0212 + +"""A version of the datetime module which *cares* about timezones. + +This module will never return a naive datetime object. This requires the module +know your local timezone, which it tries really hard to figure out. + +You can override the detection by using the datetime.tzaware.defaulttz_set +method. It the module is unable to figure out the timezone itself this method +*must* be called before the normal module is imported. If done before importing +it can also speed up the time taken to import as the defaulttz will no longer +try and do the detection. +""" + +__author__ = "tansell@google.com (Tim Ansell)" + +import calendar +import datetime +import os +import os.path +import re +import time +import warnings +import dateutil.parser +import dateutil.relativedelta +import dateutil.tz +import pytz +import pytz_abbr + + +try: + # pylint: disable-msg=C6204 + import functools +except ImportError, e: + + class functools(object): + """Fake replacement for a full functools.""" + + # pylint: disable-msg=W0613 + @staticmethod + def wraps(f, *args, **kw): + return f + + +# Need to patch pytz.utc to have a _utcoffset so you can normalize/localize +# using it. +pytz.utc._utcoffset = datetime.timedelta() + + +timedelta = datetime.timedelta + + +def _tzinfome(tzinfo): + """Gets a tzinfo object from a string. + + Args: + tzinfo: A string (or string like) object, or a datetime.tzinfo object. + + Returns: + An datetime.tzinfo object. + + Raises: + UnknownTimeZoneError: If the timezone given can't be decoded. + """ + if not isinstance(tzinfo, datetime.tzinfo): + try: + tzinfo = pytz.timezone(tzinfo) + except AttributeError: + raise pytz.UnknownTimeZoneError("Unknown timezone! %s" % tzinfo) + return tzinfo + + +# Our "local" timezone +_localtz = None + + +def localtz(): + """Get the local timezone. + + Returns: + The localtime timezone as a tzinfo object. + """ + # pylint: disable-msg=W0603 + global _localtz + if _localtz is None: + _localtz = detect_timezone() + return _localtz + + +def localtz_set(timezone): + """Set the local timezone.""" + # pylint: disable-msg=W0603 + global _localtz + _localtz = _tzinfome(timezone) + + +def detect_timezone(): + """Try and detect the timezone that Python is currently running in. + + We have a bunch of different methods for trying to figure this out (listed in + order they are attempted). + * Try TZ environment variable. + * Try and find /etc/timezone file (with timezone name). + * Try and find /etc/localtime file (with timezone data). + * Try and match a TZ to the current dst/offset/shortname. + + Returns: + The detected local timezone as a tzinfo object + + Raises: + pytz.UnknownTimeZoneError: If it was unable to detect a timezone. + """ + # First we try the TZ variable + tz = _detect_timezone_environ() + if tz is not None: + return tz + + # Second we try /etc/timezone and use the value in that + tz = _detect_timezone_etc_timezone() + if tz is not None: + return tz + + # Next we try and see if something matches the tzinfo in /etc/localtime + tz = _detect_timezone_etc_localtime() + if tz is not None: + return tz + + # Next we try and use a similiar method to what PHP does. + # We first try to search on time.tzname, time.timezone, time.daylight to + # match a pytz zone. + warnings.warn("Had to fall back to worst detection method (the 'PHP' " + "method).") + + tz = _detect_timezone_php() + if tz is not None: + return tz + + raise pytz.UnknownTimeZoneError("Unable to detect your timezone!") + + +def _detect_timezone_environ(): + if "TZ" in os.environ: + try: + return pytz.timezone(os.environ["TZ"]) + except (IOError, pytz.UnknownTimeZoneError): + warnings.warn("You provided a TZ environment value (%r) we did not " + "understand!" % os.environ["TZ"]) + + +def _detect_timezone_etc_timezone(): + if os.path.exists("/etc/timezone"): + try: + tz = file("/etc/timezone").read().strip() + try: + return pytz.timezone(tz) + except (IOError, pytz.UnknownTimeZoneError), ei: + warnings.warn("Your /etc/timezone file references a timezone (%r) that" + " is not valid (%r)." % (tz, ei)) + + # Problem reading the /etc/timezone file + except IOError, eo: + warnings.warn("Could not access your /etc/timezone file: %s" % eo) + + +def _detect_timezone_etc_localtime(): + matches = [] + if os.path.exists("/etc/localtime"): + localtime = pytz.tzfile.build_tzinfo("/etc/localtime", + file("/etc/localtime")) + + # See if we can find a "Human Name" for this.. + for tzname in pytz.all_timezones: + tz = _tzinfome(tzname) + + if dir(tz) != dir(localtime): + continue + + for attrib in dir(tz): + # Ignore functions and specials + if callable(getattr(tz, attrib)) or attrib.startswith("__"): + continue + + # This will always be different + if attrib == "zone" or attrib == "_tzinfos": + continue + + if getattr(tz, attrib) != getattr(localtime, attrib): + break + + # We get here iff break didn't happen, i.e. no meaningful attributes + # differ between tz and localtime + else: + matches.append(tzname) + + if len(matches) == 1: + return _tzinfome(matches[0]) + else: + # Warn the person about this! + warning = "Could not get a human name for your timezone: " + if len(matches) > 1: + warning += ("We detected multiple matches for your /etc/localtime. " + "(Matches where %s)" % matches) + return _tzinfome(matches[0]) + else: + warning += "We detected no matches for your /etc/localtime." + warnings.warn(warning) + + # Register /etc/localtime as the timezone loaded. + pytz._tzinfo_cache['/etc/localtime'] = localtime + return localtime + + +def _detect_timezone_php(): + tomatch = (time.tzname[0], time.timezone, time.daylight) + now = datetime.datetime.now() + + matches = [] + for tzname in pytz.all_timezones: + try: + tz = pytz.timezone(tzname) + except IOError: + continue + + try: + indst = tz.localize(now).timetuple()[-1] + + if tomatch == (tz._tzname, -tz._utcoffset.seconds, indst): + matches.append(tzname) + + # pylint: disable-msg=W0704 + except AttributeError: + pass + + if len(matches) > 1: + warnings.warn("We detected multiple matches for the timezone, choosing " + "the first %s. (Matches where %s)" % (matches[0], matches)) + return pytz.timezone(matches[0]) + + +class datetime_tz(datetime.datetime): + """An extension of the inbuilt datetime adding more functionality. + + The extra functionality includes: + * Partial parsing support (IE 2006/02/30 matches %Y/%M/%D %H:%M) + * Full integration with pytz (just give it the string of the timezone!) + * Proper support for going to/from Unix timestamps (which are in UTC!). + """ + __slots__ = ["is_dst"] + + def __new__(cls, *args, **kw): + args = list(args) + if not args: + raise TypeError("Not enough arguments given.") + + # See if we are given a tzinfo object... + tzinfo = None + if isinstance(args[-1], (datetime.tzinfo, basestring)): + tzinfo = _tzinfome(args.pop(-1)) + elif kw.get("tzinfo", None) is not None: + tzinfo = _tzinfome(kw.pop("tzinfo")) + + # Create a datetime object if we don't have one + if isinstance(args[0], datetime.datetime): + # Convert the datetime instance to a datetime object. + newargs = (list(args[0].timetuple()[0:6]) + + [args[0].microsecond, args[0].tzinfo]) + dt = datetime.datetime(*newargs) + + if tzinfo is None and dt.tzinfo is None: + raise TypeError("Must specify a timezone!") + + if tzinfo is not None and dt.tzinfo is not None: + raise TypeError("Can not give a timezone with timezone aware" + " datetime object! (Use localize.)") + else: + dt = datetime.datetime(*args, **kw) + + if dt.tzinfo is not None: + # Re-normalize the dt object + dt = dt.tzinfo.normalize(dt) + + else: + if tzinfo is None: + tzinfo = localtz() + + try: + dt = tzinfo.localize(dt, is_dst=None) + except pytz.AmbiguousTimeError: + is_dst = None + if "is_dst" in kw: + is_dst = kw.pop("is_dst") + + try: + dt = tzinfo.localize(dt, is_dst) + except IndexError: + raise pytz.AmbiguousTimeError("No such time exists!") + + newargs = list(dt.timetuple()[0:6])+[dt.microsecond, dt.tzinfo] + obj = datetime.datetime.__new__(cls, *newargs) + obj.is_dst = obj.dst() != datetime.timedelta(0) + return obj + + def asdatetime(self, naive=True): + """Return this datetime_tz as a datetime object. + + Args: + naive: Return *without* any tz info. + + Returns: + This datetime_tz as a datetime object. + """ + args = list(self.timetuple()[0:6])+[self.microsecond] + if not naive: + args.append(self.tzinfo) + return datetime.datetime(*args) + + def asdate(self): + """Return this datetime_tz as a date object. + + Returns: + This datetime_tz as a date object. + """ + return datetime.date(self.year, self.month, self.day) + + def totimestamp(self): + """Convert this datetime object back to a unix timestamp. + + The Unix epoch is the time 00:00:00 UTC on January 1, 1970. + + Returns: + Unix timestamp. + """ + return calendar.timegm(self.utctimetuple())+1e-6*self.microsecond + + def astimezone(self, tzinfo): + """Returns a version of this timestamp converted to the given timezone. + + Args: + tzinfo: Either a datetime.tzinfo object or a string (which will be looked + up in pytz. + + Returns: + A datetime_tz object in the given timezone. + """ + # Assert we are not a naive datetime object + assert self.tzinfo is not None + + tzinfo = _tzinfome(tzinfo) + + d = self.asdatetime(naive=False).astimezone(tzinfo) + return datetime_tz(d) + + # pylint: disable-msg=C6113 + def replace(self, **kw): + """Return datetime with new specified fields given as arguments. + + For example, dt.replace(days=4) would return a new datetime_tz object with + exactly the same as dt but with the days attribute equal to 4. + + Any attribute can be replaced, but tzinfo can not be set to None. + + Args: + Any datetime_tz attribute. + + Returns: + A datetime_tz object with the attributes replaced. + + Raises: + TypeError: If the given replacement is invalid. + """ + if "tzinfo" in kw: + if kw["tzinfo"] is None: + raise TypeError("Can not remove the timezone use asdatetime()") + + is_dst = None + if "is_dst" in kw: + is_dst = kw["is_dst"] + del kw["is_dst"] + else: + # Use our own DST setting.. + is_dst = self.is_dst + + replaced = self.asdatetime().replace(**kw) + + return datetime_tz(replaced, tzinfo=self.tzinfo.zone, is_dst=is_dst) + + # pylint: disable-msg=C6310 + @classmethod + def smartparse(cls, toparse, tzinfo=None): + """Method which uses dateutil.parse and extras to try and parse the string. + + Valid dates are found at: + http://labix.org/python-dateutil#head-1443e0f14ad5dff07efd465e080d1110920673d8-2 + + Other valid formats include: + "now" or "today" + "yesterday" + "tommorrow" + "5 minutes ago" + "10 hours ago" + "10h5m ago" + "start of yesterday" + "end of tommorrow" + "end of 3rd of March" + + Args: + toparse: The string to parse. + tzinfo: Timezone for the resultant datetime_tz object should be in. + (Defaults to your local timezone.) + + Returns: + New datetime_tz object. + + Raises: + ValueError: If unable to make sense of the input. + """ + # Default for empty fields are: + # year/month/day == now + # hour/minute/second/microsecond == 0 + toparse = toparse.strip() + + if tzinfo is None: + dt = cls.now() + else: + dt = cls.now(tzinfo) + + default = dt.replace(hour=0, minute=0, second=0, microsecond=0) + + # Remove "start of " and "end of " prefix in the string + if toparse.lower().startswith("end of "): + toparse = toparse[7:].strip() + + dt += datetime.timedelta(days=1) + dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) + dt -= datetime.timedelta(microseconds=1) + + default = dt + + elif toparse.lower().startswith("start of "): + toparse = toparse[9:].strip() + + dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) + default = dt + + # Handle strings with "now", "today", "yesterday", "tomorrow" and "ago". + # Need to use lowercase + toparselower = toparse.lower() + + if toparselower in ["now", "today"]: + pass + + elif toparselower == "yesterday": + dt -= datetime.timedelta(days=1) + + elif toparselower == "tommorrow": + dt += datetime.timedelta(days=1) + + elif "ago" in toparselower: + # Remove the "ago" bit + toparselower = toparselower[:-3] + # Replace all "a day and an hour" with "1 day 1 hour" + toparselower = toparselower.replace("a ", "1 ") + toparselower = toparselower.replace("an ", "1 ") + toparselower = toparselower.replace(" and ", " ") + + # Match the following + # 1 hour ago + # 1h ago + # 1 h ago + # 1 hour ago + # 2 hours ago + # Same with minutes, seconds, etc. + + tocheck = ("seconds", "minutes", "hours", "days", "weeks", "months", + "years") + result = {} + for match in re.finditer("([0-9]+)([^0-9]*)", toparselower): + amount = int(match.group(1)) + unit = match.group(2).strip() + + for bit in tocheck: + regex = "^([%s]|((%s)s?))$" % ( + bit[0], bit[:-1]) + + bitmatch = re.search(regex, unit) + if bitmatch: + result[bit] = amount + break + else: + raise ValueError("Was not able to parse date unit %r!" % unit) + + delta = dateutil.relativedelta.relativedelta(**result) + dt -= delta + + else: + # Handle strings with normal datetime format, use original case. + dt = dateutil.parser.parse(toparse, default=default.asdatetime(), + tzinfos=pytz_abbr.tzinfos) + if dt is None: + raise ValueError("Was not able to parse date!") + + if dt.tzinfo is pytz_abbr.unknown: + dt = dt.replace(tzinfo=None) + + if dt.tzinfo is None: + if tzinfo is None: + tzinfo = localtz() + dt = cls(dt, tzinfo) + else: + if isinstance(dt.tzinfo, pytz_abbr.tzabbr): + abbr = dt.tzinfo + dt = dt.replace(tzinfo=None) + dt = cls(dt, abbr.zone, is_dst=abbr.dst) + + dt = cls(dt) + + return dt + + @classmethod + def utcfromtimestamp(cls, timestamp): + """Returns a datetime object of a given timestamp (in UTC).""" + obj = datetime.datetime.utcfromtimestamp(timestamp) + obj = pytz.utc.localize(obj) + return cls(obj) + + @classmethod + def fromtimestamp(cls, timestamp): + """Returns a datetime object of a given timestamp (in local tz).""" + d = cls.utcfromtimestamp(timestamp) + return d.astimezone(localtz()) + + @classmethod + def utcnow(cls): + """Return a new datetime representing UTC day and time.""" + obj = datetime.datetime.utcnow() + obj = cls(obj, tzinfo=pytz.utc) + return obj + + @classmethod + def now(cls, tzinfo=None): + """[tz] -> new datetime with tz's local day and time.""" + obj = cls.utcnow() + if tzinfo is None: + tzinfo = localtz() + return obj.astimezone(tzinfo) + + today = now + + @staticmethod + def fromordinal(ordinal): + raise SyntaxError("Not enough information to create a datetime_tz object " + "from an ordinal. Please use datetime.date.fromordinal") + + +class iterate(object): + """Helpful iterators for working with datetime_tz objects.""" + + @staticmethod + def between(start, delta, end=None): + """Return an iterator between this date till given end point. + + Example usage: + >>> d = datetime_tz.smartparse("5 days ago") + 2008/05/12 11:45 + >>> for i in d.between(timedelta(days=1), datetime_tz.now()): + >>> print i + 2008/05/12 11:45 + 2008/05/13 11:45 + 2008/05/14 11:45 + 2008/05/15 11:45 + 2008/05/16 11:45 + + Args: + start: The date to start at. + delta: The interval to iterate with. + end: (Optional) Date to end at. If not given the iterator will never + terminate. + + Yields: + datetime_tz objects. + """ + toyield = start + while end is None or toyield < end: + yield toyield + toyield += delta + + @staticmethod + def weeks(start, end=None): + """Iterate over the weeks between the given datetime_tzs. + + Args: + start: datetime_tz to start from. + end: (Optional) Date to end at, if not given the iterator will never + terminate. + + Returns: + An iterator which generates datetime_tz objects a week apart. + """ + return iterate.between(start, datetime.timedelta(days=7), end) + + @staticmethod + def days(start, end=None): + """Iterate over the days between the given datetime_tzs. + + Args: + start: datetime_tz to start from. + end: (Optional) Date to end at, if not given the iterator will never + terminate. + + Returns: + An iterator which generates datetime_tz objects a day apart. + """ + return iterate.between(start, datetime.timedelta(days=1), end) + + @staticmethod + def hours(start, end=None): + """Iterate over the hours between the given datetime_tzs. + + Args: + start: datetime_tz to start from. + end: (Optional) Date to end at, if not given the iterator will never + terminate. + + Returns: + An iterator which generates datetime_tz objects a hour apart. + """ + return iterate.between(start, datetime.timedelta(hours=1), end) + + @staticmethod + def minutes(start, end=None): + """Iterate over the minutes between the given datetime_tzs. + + Args: + start: datetime_tz to start from. + end: (Optional) Date to end at, if not given the iterator will never + terminate. + + Returns: + An iterator which generates datetime_tz objects a minute apart. + """ + return iterate.between(start, datetime.timedelta(minutes=1), end) + + @staticmethod + def seconds(start, end=None): + """Iterate over the seconds between the given datetime_tzs. + + Args: + start: datetime_tz to start from. + end: (Optional) Date to end at, if not given the iterator will never + terminate. + + Returns: + An iterator which generates datetime_tz objects a second apart. + """ + return iterate.between(start, datetime.timedelta(minutes=1), end) + + +def _wrap_method(name): + """Wrap a method. + + Patch a method which might return a datetime.datetime to return a + datetime_tz.datetime_tz instead. + + Args: + name: The name of the method to patch + """ + method = getattr(datetime.datetime, name) + + # Have to give the second argument as method has no __module__ option. + @functools.wraps(method, ("__name__", "__doc__"), ()) + def wrapper(*args, **kw): + r = method(*args, **kw) + + if isinstance(r, datetime.datetime) and not isinstance(r, datetime_tz): + r = datetime_tz(r) + return r + + setattr(datetime_tz, name, wrapper) + +for methodname in ["__add__", "__radd__", "__rsub__", "__sub__", "combine"]: + + # Make sure we have not already got an override for this method + assert methodname not in datetime_tz.__dict__ + + _wrap_method(methodname) + + +__all__ = ['datetime_tz', 'detect_timezone', 'iterate', 'localtz', + 'localtz_set', 'timedelta', '_detect_timezone_environ', + '_detect_timezone_etc_localtime', '_detect_timezone_etc_timezone', + '_detect_timezone_php'] diff --git a/datetime_tz/pytz_abbr.py b/datetime_tz/pytz_abbr.py new file mode 100644 index 0000000..939ff4e --- /dev/null +++ b/datetime_tz/pytz_abbr.py @@ -0,0 +1,230 @@ +#!/usr/bin/python2.4 +# -*- coding: utf-8 -*- +# +# Copyright 2010 Google Inc. All Rights Reserved. +# + +""" +Common time zone acronyms/abbreviations for use with the datetime_tz module. + +*WARNING*: There are lots of caveats when using this module which are listed +below. + +CAVEAT 1: The acronyms/abbreviations are not globally unique, they are not even +unique within a region. For example, EST can mean any of, + Eastern Standard Time in Australia (which is 10 hour ahead of UTC) + Eastern Standard Time in North America (which is 5 hours behind UTC) + +Where there are two abbreviations the more popular one will appear in the all +dictionary, while the less common one will only appear in that countries region +dictionary. IE If using all, EST will be mapped to Eastern Standard Time in +North America. + +CAVEAT 2: Many of the acronyms don't map to a neat Oslon timezones. For example, +Eastern European Summer Time (EEDT) is used by many different countries in +Europe *at different times*! If the acronym does not map neatly to one zone it +is mapped to the Etc/GMT+-XX Oslon zone. This means that any date manipulations +can end up with idiot things like summer time in the middle of winter. + +CAVEAT 3: The Summer/Standard time difference is really important! For an hour +each year it is needed to determine which time you are actually talking about. + 2002-10-27 01:20:00 EST != 2002-10-27 01:20:00 EDT +""" + +import datetime +import pytz +import pytz.tzfile + + +class tzabbr(datetime.tzinfo): + """A timezone abbreviation. + + *WARNING*: This is not a tzinfo implementation! Trying to use this as tzinfo + object will result in failure. We inherit from datetime.tzinfo so we can get + through the dateutil checks. + """ + pass + + +# A "marker" tzinfo object which is used to signify an unknown timezone. +unknown = datetime.tzinfo(0) + + +regions = {'all': {}, 'military': {}} +# Create a special alias for the all and military regions +all = regions['all'] +military = regions['military'] + + +def tzabbr_register(abbr, name, region, zone, dst): + """Register a new timezone abbreviation in the global registry. + + If another abbreviation with the same name has already been registered it new + abbreviation will only be registered in region specific dictionary. + """ + newabbr = tzabbr() + newabbr.abbr = abbr + newabbr.name = name + newabbr.region = region + newabbr.zone = zone + newabbr.dst = dst + + if abbr not in all: + all[abbr] = newabbr + + if not region in regions: + regions[region] = {} + + assert abbr not in regions[region] + regions[region][abbr] = newabbr + + +def tzinfos_create(use_region): + abbrs = regions[use_region] + + def tzinfos(abbr, offset): + if abbr: + if abbr in abbrs: + result = abbrs[abbr] + if offset: + # FIXME: Check the offset matches the abbreviation we just selected. + pass + return result + else: + raise ValueError, "Unknown timezone found %s" % abbr + if offset == 0: + return pytz.utc + if offset: + return pytz.FixedOffset(offset/60) + return unknown + + return tzinfos + + +# Create a special alias for the all tzinfos +tzinfos = tzinfos_create('all') + + +# Create the abbreviations. +# *WARNING*: Order matters! +tzabbr_register("A", u"Alpha Time Zone", u"Military", "Etc/GMT-1", False) +tzabbr_register("ACDT", u"Australian Central Daylight Time", u"Australia", + "Australia/Adelaide", True) +tzabbr_register("ACST", u"Australian Central Standard Time", u"Australia", + "Australia/Adelaide", False) +tzabbr_register("ADT", u"Atlantic Daylight Time", u"North America", + "America/Halifax", True) +tzabbr_register("AEDT", u"Australian Eastern Daylight Time", u"Australia", + "Australia/Sydney", True) +tzabbr_register("AEST", u"Australian Eastern Standard Time", u"Australia", + "Australia/Sydney", False) +tzabbr_register("AKDT", u"Alaska Daylight Time", u"North America", + "US/Alaska", True) +tzabbr_register("AKST", u"Alaska Standard Time", u"North America", + "US/Alaska", False) +tzabbr_register("AST", u"Atlantic Standard Time", u"North America", + "America/Halifax", False) +tzabbr_register("AWDT", u"Australian Western Daylight Time", u"Australia", + "Australia/West", True) +tzabbr_register("AWST", u"Australian Western Standard Time", u"Australia", + "Australia/West", False) +tzabbr_register("B", u"Bravo Time Zone", u"Military", "Etc/GMT-2", False) +tzabbr_register("BST", u"British Summer Time", u"Europe", "Europe/London", True) +tzabbr_register("C", u"Charlie Time Zone", u"Military", "Etc/GMT-2", False) +tzabbr_register("CDT", u"Central Daylight Time", u"North America", + "US/Central", True) +tzabbr_register("CEDT", u"Central European Daylight Time", u"Europe", + "Etc/GMT+2", True) +tzabbr_register("CEST", u"Central European Summer Time", u"Europe", + "Etc/GMT+2", True) +tzabbr_register("CET", u"Central European Time", u"Europe", "Etc/GMT+1", False) +tzabbr_register("CST", u"Central Standard Time", u"North America", + "US/Central", False) +tzabbr_register("CXT", u"Christmas Island Time", u"Australia", + "Indian/Christmas", False) +tzabbr_register("D", u"Delta Time Zone", u"Military", "Etc/GMT-2", False) +tzabbr_register("E", u"Echo Time Zone", u"Military", "Etc/GMT-2", False) +tzabbr_register("EDT", u"Eastern Daylight Time", u"North America", + "US/Eastern", True) +tzabbr_register("EEDT", u"Eastern European Daylight Time", u"Europe", + "Etc/GMT+3", True) +tzabbr_register("EEST", u"Eastern European Summer Time", u"Europe", + "Etc/GMT+3", True) +tzabbr_register("EET", u"Eastern European Time", u"Europe", "Etc/GMT+2", False) +tzabbr_register("EST", u"Eastern Standard Time", u"North America", + "US/Eastern", False) +tzabbr_register("F", u"Foxtrot Time Zone", u"Military", "Etc/GMT-6", False) +tzabbr_register("G", u"Golf Time Zone", u"Military", "Etc/GMT-7", False) +tzabbr_register("GMT", u"Greenwich Mean Time", u"Europe", pytz.utc, False) +tzabbr_register("H", u"Hotel Time Zone", u"Military", "Etc/GMT-8", False) +#tzabbr_register("HAA", u"Heure Avancée de l'Atlantique", u"North America", u"UTC - 3 hours") +#tzabbr_register("HAC", u"Heure Avancée du Centre", u"North America", u"UTC - 5 hours") +tzabbr_register("HADT", u"Hawaii-Aleutian Daylight Time", u"North America", + "Pacific/Honolulu", True) +#tzabbr_register("HAE", u"Heure Avancée de l'Est", u"North America", u"UTC - 4 hours") +#tzabbr_register("HAP", u"Heure Avancée du Pacifique", u"North America", u"UTC - 7 hours") +#tzabbr_register("HAR", u"Heure Avancée des Rocheuses", u"North America", u"UTC - 6 hours") +tzabbr_register("HAST", u"Hawaii-Aleutian Standard Time", u"North America", + "Pacific/Honolulu", False) +#tzabbr_register("HAT", u"Heure Avancée de Terre-Neuve", u"North America", u"UTC - 2:30 hours") +#tzabbr_register("HAY", u"Heure Avancée du Yukon", u"North America", u"UTC - 8 hours") +tzabbr_register("HDT", u"Hawaii Daylight Time", u"North America", + "Pacific/Honolulu", True) +#tzabbr_register("HNA", u"Heure Normale de l'Atlantique", u"North America", u"UTC - 4 hours") +#tzabbr_register("HNC", u"Heure Normale du Centre", u"North America", u"UTC - 6 hours") +#tzabbr_register("HNE", u"Heure Normale de l'Est", u"North America", u"UTC - 5 hours") +#tzabbr_register("HNP", u"Heure Normale du Pacifique", u"North America", u"UTC - 8 hours") +#tzabbr_register("HNR", u"Heure Normale des Rocheuses", u"North America", u"UTC - 7 hours") +#tzabbr_register("HNT", u"Heure Normale de Terre-Neuve", u"North America", u"UTC - 3:30 hours") +#tzabbr_register("HNY", u"Heure Normale du Yukon", u"North America", u"UTC - 9 hours") +tzabbr_register("HST", u"Hawaii Standard Time", u"North America", + "Pacific/Honolulu", False) +tzabbr_register("I", u"India Time Zone", u"Military", "Etc/GMT-9", False) +tzabbr_register("IST", u"Irish Summer Time", u"Europe", "Europe/Dublin", True) +tzabbr_register("K", u"Kilo Time Zone", u"Military", "Etc/GMT-10", False) +tzabbr_register("L", u"Lima Time Zone", u"Military", "Etc/GMT-11", False) +tzabbr_register("M", u"Mike Time Zone", u"Military", "Etc/GMT-12", False) +tzabbr_register("MDT", u"Mountain Daylight Time", u"North America", + "US/Mountain", True) +#tzabbr_register("MESZ", u"Mitteleuroäische Sommerzeit", u"Europe", u"UTC + 2 hours") +#tzabbr_register("MEZ", u"Mitteleuropäische Zeit", u"Europe", u"UTC + 1 hour") +tzabbr_register("MSD", u"Moscow Daylight Time", u"Europe", + "Europe/Moscow", True) +tzabbr_register("MSK", u"Moscow Standard Time", u"Europe", + "Europe/Moscow", False) +tzabbr_register("MST", u"Mountain Standard Time", u"North America", + "US/Mountain", False) +tzabbr_register("N", u"November Time Zone", u"Military", "Etc/GMT+1", False) +tzabbr_register("NDT", u"Newfoundland Daylight Time", u"North America", + "America/St_Johns", True) +tzabbr_register("NFT", u"Norfolk (Island) Time", u"Australia", + "Pacific/Norfolk", False) +tzabbr_register("NST", u"Newfoundland Standard Time", u"North America", + "America/St_Johns", False) +tzabbr_register("O", u"Oscar Time Zone", u"Military", "Etc/GMT+2", False) +tzabbr_register("P", u"Papa Time Zone", u"Military", "Etc/GMT+3", False) +tzabbr_register("PDT", u"Pacific Daylight Time", u"North America", + "US/Pacific", True) +tzabbr_register("PST", u"Pacific Standard Time", u"North America", + "US/Pacific", False) +tzabbr_register("Q", u"Quebec Time Zone", u"Military", "Etc/GMT+4", False) +tzabbr_register("R", u"Romeo Time Zone", u"Military", "Etc/GMT+5", False) +tzabbr_register("S", u"Sierra Time Zone", u"Military", "Etc/GMT+6", False) +tzabbr_register("T", u"Tango Time Zone", u"Military", "Etc/GMT+7", False) +tzabbr_register("U", u"Uniform Time Zone", u"Military", "Etc/GMT+8", False) +tzabbr_register("UTC", u"Coordinated Universal Time", u"Europe", + pytz.utc, False) +tzabbr_register("V", u"Victor Time Zone", u"Military", "Etc/GMT+9", False) +tzabbr_register("W", u"Whiskey Time Zone", u"Military", "Etc/GMT+10", False) +tzabbr_register("WDT", u"Western Daylight Time", u"Australia", + "Australia/West", True) +tzabbr_register("WEDT", u"Western European Daylight Time", u"Europe", + "Etc/GMT+1", True) +tzabbr_register("WEST", u"Western European Summer Time", u"Europe", + "Etc/GMT+1", True) +tzabbr_register("WET", u"Western European Time", u"Europe", pytz.utc, False) +tzabbr_register("WST", u"Western Standard Time", u"Australia", + "Australia/West", False) +tzabbr_register("X", u"X-ray Time Zone", u"Military", "Etc/GMT+11", False) +tzabbr_register("Y", u"Yankee Time Zone", u"Military", "Etc/GMT+12", False) +tzabbr_register("Z", u"Zulu Time Zone", u"Military", pytz.utc, False) diff --git a/nilmdb/__init__.py b/nilmdb/__init__.py index efa9cae..d374392 100644 --- a/nilmdb/__init__.py +++ b/nilmdb/__init__.py @@ -3,3 +3,9 @@ from .nilmdb import NilmDB, StreamError from .server import Server from .client import Client, ClientError, ServerError + +import layout +import serializer +import fixup +import cmdline +import timestamper diff --git a/nilmdb/client.py b/nilmdb/client.py index 6eea813..0d32ba2 100644 --- a/nilmdb/client.py +++ b/nilmdb/client.py @@ -41,6 +41,7 @@ class ServerError(NilmCommError): pass class MyCurl(object): + """Curl wrapper for HTTP client requests""" def __init__(self, baseurl = ""): """If baseurl is supplied, all other functions that take a URL can be given a relative URL instead.""" @@ -97,6 +98,7 @@ class MyCurl(object): return json.loads(body_str) class Client(object): + """Main client interface to the Nilm database.""" def __init__(self, url): self.curl = MyCurl(url) @@ -152,16 +154,3 @@ class Client(object): params = { "path": path } return self.curl.getjson("stream/insert", params) - # def test_insert(self): - # # invalid path first - # with assert_raises(HTTPError) as e: - # getjson("/stream/insert?path=/newton/") - # eq_(e.exception.code, 404) - - # # XXX TODO - # with assert_raises(HTTPError) as e: - # getjson("/stream/insert?path=/newton/prep") - # eq_(e.exception.code, 501) - - - diff --git a/nilmdb/server.py b/nilmdb/server.py index 7ed2b92..bdc7d58 100644 --- a/nilmdb/server.py +++ b/nilmdb/server.py @@ -153,7 +153,7 @@ class Stream(NilmApp): raise cherrypy.HTTPError("404 Not Found", "No such stream") layout = streams[0][1] - raise cherrypy.HTTPError("501 Not Implemented", layout) + raise cherrypy.HTTPError("501 Not Implemented", "Layout is: " + layout) class Exiter(object): """App that exits the server, for testing""" diff --git a/nilmdb/timestamper.py b/nilmdb/timestamper.py new file mode 100644 index 0000000..107b00c --- /dev/null +++ b/nilmdb/timestamper.py @@ -0,0 +1,58 @@ +"""File-like objects that add timestamps to the input lines""" + +from __future__ import absolute_import +from nilmdb.printf import * + +import time +import os +import datetime_tz + +class Timestamper(object): + """A file-like object that adds timestamps to lines of an input file.""" + def __init__(self, file, ts_iter): + """file: filename, or another file-like object + ts_iter: iterator that returns a timestamp string for + each line of the file""" + if isinstance(file, basestring): + self.file = open(file, "r") + else: + self.file = file + self.ts_iter = ts_iter + + def close(self): + self.file.close() + + def readline(self, *args): + line = self.file.readline(*args) + if line: + line = self.ts_iter.next() + line + return line + + def __iter__(self): + return self + + def next(self): + result = self.readline() + if not result: + raise StopIteration + return result + +class TimestamperRate(Timestamper): + """Timestamper that uses a start time and a fixed rate""" + def __init__(self, file, start_timestamp, rate_hz): + def iterator(start, rate): + n = 0 + rate = float(rate) + while True: + yield sprintf("%.6f ", start + n / rate) + n += 1 + Timestamper.__init__(self, file, iterator(start_timestamp, rate_hz)) + +class TimestamperNow(Timestamper): + """Timestamper that uses current time""" + def __init__(self, file): + def iterator(): + while True: + now = datetime_tz.datetime_tz.utcnow().totimestamp() + yield sprintf("%.6f ", now) + Timestamper.__init__(self, file, iterator()) diff --git a/nilmtool.py b/nilmtool.py index 6aa6830..0e49ae3 100755 --- a/nilmtool.py +++ b/nilmtool.py @@ -1,5 +1,6 @@ #!/usr/bin/python import nilmdb +import sys nilmdb.cmdline.run(args = sys.argv[1:]) diff --git a/setup.cfg b/setup.cfg index b6c200d..f91c13e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,8 @@ cover-erase=1 ##cover-branches=1 # need nose 1.1.3 for this stop=1 verbosity=2 -#tests=tests/test_nilmdb.py +tests=tests/test_timestamper.py +#tests=tests/test_serializer.py #tests=tests/test_client.py:TestClient.test_client_nilmdb #with-profile=1 #profile-sort=time diff --git a/tests/test_client.py b/tests/test_client.py index 708fba9..79a02a1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,8 +1,9 @@ import nilmdb from nilmdb.printf import * - from nilmdb.client import ClientError, ServerError +import datetime_tz + from nose.tools import * from nose.tools import assert_raises import json @@ -50,7 +51,7 @@ def teardown_module(): class TestClient(object): - def test_client_basic(self): + def test_client_1_basic(self): # Test a fake host client = nilmdb.Client(url = "http://localhost:1/") with assert_raises(nilmdb.ServerError): @@ -67,7 +68,7 @@ class TestClient(object): eq_(distutils.version.StrictVersion(version), distutils.version.StrictVersion(test_server.version)) - def test_client_nilmdb(self): + def test_client_2_nilmdb(self): # Basic stream tests, like those in test_nilmdb:test_stream client = nilmdb.Client(url = "http://localhost:12380/") @@ -122,15 +123,22 @@ class TestClient(object): with assert_raises(ClientError): client.stream_update_metadata("/newton/prep", [1,2,3]) - def test_client_insert(self): + def test_client_3_insert(self): client = nilmdb.Client(url = "http://localhost:12380/") -# with open("data/prep-20120323T1000") as prepdata: -# client.stream_insert("/newton/prep", prepdata) # dunno - - -# ret = client.stream_insert - + datetime_tz.localtz_set("America/New_York") + + with open("tests/data/prep-20120323T1000") as prepdata: + start = datetime_tz.datetime_tz.smartparse("20120323T1000") + client.stream_insert_nots("/newton/prep", prepdata, + start, rate) + + # insert with start/end + with open("tests/data/prep-20120323T1002") as prepdata: + start = datetime_tz.datetime_tz.smartparse("20120323T1002") + end = start + datetime_tz.timedelta(minutes=2) + client.stream_insert_untimed("/newton/prep", prepdata, + start = start, end = end) # TODO: Just start writing the client side of this as if it works. # Then fill things in to make it so! diff --git a/tests/test_timestamper.py b/tests/test_timestamper.py new file mode 100644 index 0000000..b1aee17 --- /dev/null +++ b/tests/test_timestamper.py @@ -0,0 +1,30 @@ +import nilmdb +from nilmdb.printf import * + +import datetime_tz + +from nose.tools import * +from nose.tools import assert_raises +import os +import sys + +def eq_(a, b): + if not a == b: + raise AssertionError("%r != %r" % (a, b)) + +def ne_(a, b): + if not a != b: + raise AssertionError("unexpected %r == %r" % (a, b)) + +class TestTimestamper(object): + + def test_timestamper(self): + now = datetime_tz.datetime_tz.now().totimestamp() + +# ts = nilmdb.timestamper.TimestamperRate("/tmp/foo", now, 8000) +# for line in ts: +# print line.strip() + + ts = nilmdb.timestamper.TimestamperNow("/tmp/foo") + for line in ts: + print line.strip()