nilmdb/tests/test_cmdline.py

import nilmdb
from nilmdb.printf import *
import nilmdb.cmdline

from nose.tools import *
from nose.tools import assert_raises
import itertools
import datetime_tz
import os
import shutil
import sys
import threading
import urllib2
from urllib2 import urlopen, HTTPError
import Queue
import cStringIO
import shlex

from test_helpers import *

testdb = "tests/cmdline-testdb"

def server_start(response_size = None):
    global test_server, test_db
    # Start web app on a custom port
    test_db = nilmdb.NilmDB(testdb, sync = False, response_size = response_size)
    test_server = nilmdb.Server(test_db, host = "127.0.0.1",
                                port = 12380, stoppable = False,
                                fast_shutdown = True,
                                force_traceback = False)
    test_server.start(blocking = False)

def server_stop():
    global test_server, test_db
    # Close web app
    test_server.stop()
    test_db.close()

def setup_module():
    global test_server, test_db
    # Clear out DB
    recursive_unlink(testdb)
    server_start()

def teardown_module():
    server_stop()

class TestCmdline(object):

    def run(self, arg_string, infile=None, outfile=None):
        """Run a cmdline client with the specified argument string,
        passing the given input.  Returns a tuple with the output and
        exit code"""
        class stdio_wrapper:
            def __init__(self, stdin, stdout, stderr):
                self.io = (stdin, stdout, stderr)
            def __enter__(self):
                self.saved = ( sys.stdin, sys.stdout, sys.stderr )
                ( sys.stdin, sys.stdout, sys.stderr ) = self.io
            def __exit__(self, type, value, traceback):
                ( sys.stdin, sys.stdout, sys.stderr ) = self.saved
        # Empty input if none provided
        if infile is None:
            infile = cStringIO.StringIO("")
        # Capture stderr
        errfile = cStringIO.StringIO()
        if outfile is None:
            # If no output file, capture stdout with stderr
            outfile = errfile
        with stdio_wrapper(infile, outfile, errfile) as s:
            try:
                nilmdb.cmdline.Cmdline(shlex.split(arg_string)).run()
                sys.exit(0)
            except SystemExit as e:
                exitcode = e.code
        captured = outfile.getvalue()
        self.captured = captured
        self.exitcode = exitcode

    def ok(self, arg_string, infile = None):
        self.run(arg_string, infile)
        if self.exitcode != 0:
            self.dump()
            eq_(self.exitcode, 0)

    def fail(self, arg_string, infile = None):
        self.run(arg_string, infile)
        if self.exitcode == 0:
            self.dump()
            ne_(self.exitcode, 0)

    def contain(self, checkstring):
        in_(checkstring, self.captured)

    def match(self, checkstring):
        eq_(checkstring, self.captured)

    def dump(self):
        printf("-----dump start-----\n%s-----dump end-----\n", self.captured)

    def test_cmdline_1_basic(self):

        # help
        self.ok("--help")
        self.contain("usage:")

        # fail for no args
        self.fail("")

        # fail for no such option
        self.fail("--nosuchoption")

        # fail for bad command
        self.fail("badcommand")

        # try some URL constructions
        self.fail("--url http://nosuchurl/ info")
        self.contain("Couldn't resolve host 'nosuchurl'")

        self.fail("--url nosuchurl info")
        self.contain("Couldn't resolve host 'nosuchurl'")

        self.fail("-u nosuchurl/foo info")
        self.contain("Couldn't resolve host 'nosuchurl'")

        self.fail("-u localhost:0 info")
        self.contain("couldn't connect to host")

        self.ok("-u localhost:12380 info")
        self.ok("info")

    def test_cmdline_2_info(self):
        self.ok("info")
        self.contain("Server URL: http://localhost:12380/")
        self.contain("Server version: " + test_server.version)

    def test_cmdline_3_createlist(self):
        # Basic stream tests, like those in test_client.

        # No streams
        self.ok("list")
        self.match("")

        # Bad paths
        self.fail("create foo/bar/baz PrepData")
        self.contain("paths must start with /")

        self.fail("create /foo PrepData")
        self.contain("invalid path")

        # Bad layout type
        self.fail("create /newton/prep NoSuchLayout")
        self.contain("no such layout")

        # Create a few streams
        self.ok("create /newton/prep PrepData")
        self.ok("create /newton/raw RawData")
        self.ok("create /newton/zzz/rawnotch RawNotchedData")

        # Verify we got those 3 streams
        self.ok("list")
        self.match("/newton/prep PrepData\n"
                   "/newton/raw RawData\n"
                   "/newton/zzz/rawnotch RawNotchedData\n")

        # Match just one type or one path
        self.ok("list --path /newton/raw")
        self.match("/newton/raw RawData\n")

        self.ok("list --layout RawData")
        self.match("/newton/raw RawData\n")

        # Wildcard matches
        self.ok("list --layout Raw*")
        self.match("/newton/raw RawData\n"
                   "/newton/zzz/rawnotch RawNotchedData\n")

        self.ok("list --path *zzz* --layout Raw*")
        self.match("/newton/zzz/rawnotch RawNotchedData\n")

        self.ok("list --path *zzz* --layout Prep*")
        self.match("")

    def test_cmdline_4_metadata(self):
        # Set / get metadata
        self.fail("metadata")
        self.fail("metadata --get")

        self.ok("metadata /newton/prep")
        self.match("")

        self.ok("metadata /newton/raw --get")
        self.match("")

        self.ok("metadata /newton/prep --set "
                "'description=The Data' "
                "v_scale=1.234")
        self.ok("metadata /newton/raw --update "
                "'description=The Data'")
        self.ok("metadata /newton/raw --update "
                "v_scale=1.234")

        # various parsing tests
        self.ok("metadata /newton/raw --update foo=")
        self.fail("metadata /newton/raw --update =bar")
        self.fail("metadata /newton/raw --update foo==bar")
        self.fail("metadata /newton/raw --update foo;bar")

        # errors
        self.fail("metadata /newton/nosuchstream foo=bar")
        self.contain("unrecognized arguments")
        self.fail("metadata /newton/nosuchstream")
        self.contain("No stream at path")
        self.fail("metadata /newton/nosuchstream --set foo=bar")
        self.contain("No stream at path")

        self.ok("metadata /newton/prep")
        self.match("description=The Data\nv_scale=1.234\n")

        self.ok("metadata /newton/prep --get")
        self.match("description=The Data\nv_scale=1.234\n")

        self.ok("metadata /newton/prep --get descr")
        self.match("descr=\n")

        self.ok("metadata /newton/prep --get description")
        self.match("description=The Data\n")

        self.ok("metadata /newton/prep --get description v_scale")
        self.match("description=The Data\nv_scale=1.234\n")

        self.ok("metadata /newton/prep --set "
                "'description=The Data'")

        self.ok("metadata /newton/prep --get")
        self.match("description=The Data\n")

        self.fail("metadata /newton/nosuchpath")
        self.contain("No stream at path /newton/nosuchpath")

    def test_cmdline_5_parsetime(self):
        cmd = nilmdb.cmdline.Cmdline(None)
        test = datetime_tz.datetime_tz.now()
        eq_(cmd.parse_time(str(test)), test)
        test = datetime_tz.datetime_tz.smartparse("20120405 1400-0400")
        eq_(cmd.parse_time("hi there 20120405 1400-0400 testing! 123"), test)
        eq_(cmd.parse_time("20120405 1800 UTC"), test)
        eq_(cmd.parse_time("20120405 1400-0400 UTC"), test)
        with assert_raises(ValueError):
            print cmd.parse_time("20120405 1400-9999")
        with assert_raises(ValueError):
            print cmd.parse_time("hello")
        with assert_raises(ValueError):
            print cmd.parse_time("-")
        with assert_raises(ValueError):
            print cmd.parse_time("")
        with assert_raises(ValueError):
            print cmd.parse_time("14:00")
        eq_(cmd.parse_time("snapshot-20120405-140000.raw.gz"), test)
        eq_(cmd.parse_time("prep-20120405T1400"), test)

    def test_cmdline_6_insert(self):
        self.ok("insert --help")

        self.fail("insert /foo/bar baz qwer")
        self.contain("Error getting stream info")

        self.fail("insert /newton/prep baz qwer")
        self.match("Error opening input file baz\n")

        self.fail("insert /newton/prep")
        self.contain("Error extracting time")

        self.fail("insert --start 19801205 /newton/prep 1 2 3 4")
        self.contain("--start can only be used with one input file")

        # insert pre-timestamped data, from stdin
        os.environ['TZ'] = "UTC"
        with open("tests/data/prep-20120323T1004-timestamped") as input:
            self.ok("insert --none /newton/prep", input)

        # insert data with normal timestamper from filename
        os.environ['TZ'] = "UTC"
        self.ok("insert /newton/prep "
                "tests/data/prep-20120323T1000 "
                "tests/data/prep-20120323T1002")

        # overlap
        os.environ['TZ'] = "UTC"
        self.fail("insert /newton/prep "
                  "tests/data/prep-20120323T1004")
        self.contain("overlap")

        # Just to help test more situations -- stop and restart
        # the server now.  This tests nilmdb's interval caching,
        # at the very least.
        server_stop()
        server_start()

        # still an overlap if we specify a different start
        os.environ['TZ'] = "America/New_York"
        self.fail("insert --start '03/23/2012 06:05:00' /newton/prep "
                  "tests/data/prep-20120323T1004")
        self.contain("overlap")

        # wrong format
        os.environ['TZ'] = "UTC"
        self.fail("insert /newton/raw "
                  "tests/data/prep-20120323T1004")
        self.contain("Error parsing input data")

        # empty data does nothing
        self.ok("insert --start '03/23/2012 06:05:00' /newton/prep "
                "/dev/null")

        # bad start time
        self.fail("insert --start 'whatever' /newton/prep /dev/null")

    def test_cmdline_7_detail(self):
        # Just count the number of lines, it's probably fine
        self.ok("list --detail")
        eq_(self.captured.count('\n'), 11)

        self.ok("list --detail --path *prep")
        eq_(self.captured.count('\n'), 7)

        self.ok("list --detail --path *prep --start='23 Mar 2012 10:02'")
        eq_(self.captured.count('\n'), 5)

        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05'")
        eq_(self.captured.count('\n'), 3)

        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15'")
        eq_(self.captured.count('\n'), 2)
        self.contain("10:05:15.000")

        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'")
        eq_(self.captured.count('\n'), 2)
        self.contain("10:05:15.500")

        self.ok("list --detail --path *prep --start='23 Mar 2012 19:05:15.50'")
        eq_(self.captured.count('\n'), 2)
        self.contain("no intervals")

        self.ok("list --detail --path *prep --start='23 Mar 2012 10:05:15.50'"
                + " --end='23 Mar 2012 10:05:15.50'")
        eq_(self.captured.count('\n'), 2)
        self.contain("10:05:15.500")

        self.ok("list --detail")
        eq_(self.captured.count('\n'), 11)

    def test_cmdline_8_extract(self):
        # nonexistent stream
        self.fail("extract /no/such/foo --start 2000-01-01 --end 2020-01-01")
        self.contain("Error getting stream info")

        # full dump
        #self.ok("extract -a /newton/prep --start 2000-01-01 --end 2020-01-01")
        #self.dump()

    def test_cmdline_9_truncated(self):
        # Test truncated responses by overriding the nilmdb response_size
        server_stop()
        server_start(response_size = 300)

        self.ok("list --detail")
        eq_(self.captured.count('\n'), 11)