Clean up some pylint warnings and potential errors

.pylintrc

@@ -0,0 +1,250 @@
+# -*- conf -*-
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=datetime_tz
+
+# Pickle collected data for later comparisons.
+persistent=no
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+disable=C0111,R0903,R0201,R0914,R0912,W0142,W0703,W0702
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html
+output-format=parseable
+
+# Include message's id in output
+include-ids=yes
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (RP0004).
+comment=no
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, add a predefined set of Zope acquired attributes
+# to generated-members.
+zope=no
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E0201 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=80
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the beginning of the name of dummy variables
+# (i.e. not used).
+dummy-variables-rgx=_|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=apply,input
+
+# Regular expression which should only match correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression which should only match correct module level names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|version)$
+
+# Regular expression which should only match correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression which should only match correct function names
+function-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct method names
+method-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct argument names
+argument-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct variable names
+variable-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Regular expression which should only match functions or classes name which do
+# not require a docstring
+no-docstring-rgx=__.*__
+
+
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defines in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branchs=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception

Makefile

@@ -1,7 +1,7 @@
 all: test
 
 lint:
-	pylint -f parseable nilmdb
+	pylint --rcfile=.pylintrc nilmdb
 
 test:
 	python runtests.py

nilmdb/__init__.py

@@ -1,4 +1,4 @@
 """Main NilmDB import"""
 
-from server import NilmDB, Server
-from client import Client
+from nilmdb.server import NilmDB, Server
+from nilmdb.client import Client

nilmdb/client/__init__.py

@@ -1,4 +1,4 @@
 """nilmdb.client"""
 
 from .client import Client
-from .errors import *
+from .errors import ClientError, ServerError, Error

nilmdb/client/client.py

@@ -5,19 +5,14 @@
 import nilmdb
 import nilmdb.utils
 import nilmdb.client.httpclient
-from nilmdb.utils.printf import *
 
 import time
-import sys
-import re
-import os
 import simplejson as json
-import itertools
 
 version = "1.0"
 
 def float_to_string(f):
-    # Use repr to maintain full precision in the string output.
+    """Use repr to maintain full precision in the string output."""
     return repr(float(f))
 
 class Client(object):
@@ -151,6 +146,8 @@ class Client(object):
         block_data = ""
         block_start = start
         result = None
+        line = None
+        nextline = None
         for (line, nextline) in nilmdb.utils.misc.pairwise(data):
             # If we don't have a starting time, extract it from the first line
             if block_start is None:

nilmdb/client/httpclient.py

@@ -2,13 +2,8 @@
 
 import nilmdb
 import nilmdb.utils
-from nilmdb.utils.printf import *
-from nilmdb.client.errors import *
+from nilmdb.client.errors import ClientError, ServerError, Error
 
-import time
-import sys
-import re
-import os
 import simplejson as json
 import urlparse
 import pycurl

nilmdb/cmdline/cmdline.py

@@ -4,7 +4,6 @@ import nilmdb
 from nilmdb.utils.printf import *
 from nilmdb.utils import datetime_tz
 
-import dateutil.parser
 import sys
 import re
 import argparse

nilmdb/cmdline/create.py

@@ -1,9 +1,7 @@
 from nilmdb.utils.printf import *
 import nilmdb
 import nilmdb.client
-import textwrap
 
-from argparse import ArgumentDefaultsHelpFormatter as def_form
 from argparse import RawDescriptionHelpFormatter as raw_form
 
 def setup(self, sub):

nilmdb/cmdline/extract.py

@@ -1,8 +1,6 @@
 from __future__ import print_function
 from nilmdb.utils.printf import *
-import nilmdb
 import nilmdb.client
-import sys
 
 def setup(self, sub):
     cmd = sub.add_parser("extract", help="Extract data",

nilmdb/cmdline/insert.py

@@ -53,8 +53,6 @@ def cmd_insert(self):
     if len(streams) != 1:
         self.die("error getting stream info for path %s", self.args.path)
 
-    layout = streams[0][1]
-
     if self.args.start and len(self.args.file) != 1:
         self.die("error: --start can only be used with one input file")
 
@@ -93,7 +91,7 @@ def cmd_insert(self):
 
         # Insert the data
         try:
-            result = self.client.stream_insert(self.args.path, ts)
+            self.client.stream_insert(self.args.path, ts)
         except nilmdb.client.Error as e:
             # TODO: It would be nice to be able to offer better errors
             # here, particularly in the case of overlap, which just shows

nilmdb/cmdline/list.py

@@ -1,6 +1,4 @@
 from nilmdb.utils.printf import *
-import nilmdb
-import nilmdb.client
 
 import fnmatch
 import argparse

nilmdb/cmdline/remove.py

@@ -1,7 +1,6 @@
 from nilmdb.utils.printf import *
 import nilmdb
 import nilmdb.client
-import sys
 
 def setup(self, sub):
     cmd = sub.add_parser("remove", help="Remove data",

nilmdb/server/__init__.py

@@ -1,5 +1,7 @@
 """nilmdb.server"""
 
+from __future__ import absolute_import
+
 # Try to set up pyximport to automatically rebuild Cython modules.  If
 # this doesn't work, it's OK, as long as the modules were built externally.
 # (e.g. python setup.py build_ext --inplace)
@@ -10,6 +12,6 @@ try:
 except: # pragma: no cover
     pass
 
-from .nilmdb import NilmDB
-from .server import Server
-from .errors import *
+from nilmdb.server.nilmdb import NilmDB
+from nilmdb.server.server import Server
+from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

nilmdb/server/bulkdata.py

@@ -8,10 +8,8 @@ import nilmdb
 from nilmdb.utils.printf import *
 
 import os
-import sys
 import cPickle as pickle
 import struct
-import fnmatch
 import mmap
 import re
 
@@ -91,8 +89,7 @@ class BulkData(object):
                 "float32": 'f',
                 "float64": 'd',
                 }
-            for n in range(layout.count):
-                struct_fmt += struct_mapping[layout.datatype]
+            struct_fmt += struct_mapping[layout.datatype] * layout.count
         except KeyError:
             raise ValueError("no such layout, or bad data types")
 
@@ -185,12 +182,12 @@ class Table(object):
         packer = struct.Struct(struct_fmt)
         rows_per_file = max(file_size // packer.size, 1)
 
-        format = { "rows_per_file": rows_per_file,
-                   "files_per_dir": files_per_dir,
-                   "struct_fmt": struct_fmt,
-                   "version": 1 }
+        fmt = { "rows_per_file": rows_per_file,
+                "files_per_dir": files_per_dir,
+                "struct_fmt": struct_fmt,
+                "version": 1 }
         with open(os.path.join(root, "_format"), "wb") as f:
-            pickle.dump(format, f, 2)
+            pickle.dump(fmt, f, 2)
 
     # Normal methods
     def __init__(self, root):
@@ -199,15 +196,15 @@ class Table(object):
 
         # Load the format and build packer
         with open(os.path.join(self.root, "_format"), "rb") as f:
-            format = pickle.load(f)
+            fmt = pickle.load(f)
 
-        if format["version"] != 1: # pragma: no cover (just future proofing)
-            raise NotImplementedError("version " + format["version"] +
+        if fmt["version"] != 1: # pragma: no cover (just future proofing)
+            raise NotImplementedError("version " + fmt["version"] +
                                       " bulk data store not supported")
 
-        self.rows_per_file = format["rows_per_file"]
-        self.files_per_dir = format["files_per_dir"]
-        self.packer = struct.Struct(format["struct_fmt"])
+        self.rows_per_file = fmt["rows_per_file"]
+        self.files_per_dir = fmt["files_per_dir"]
+        self.packer = struct.Struct(fmt["struct_fmt"])
         self.file_size = self.packer.size * self.rows_per_file
 
         # Find nrows
@@ -278,7 +275,7 @@ class Table(object):
 
     # Cache open files
     @nilmdb.utils.lru_cache(size = fd_cache_size,
-                            keys = slice(0,3), # exclude newsize
+                            keys = slice(0, 3), # exclude newsize
                             onremove = lambda x: x.close())
     def mmap_open(self, subdir, filename, newsize = None):
         """Open and map a given 'subdir/filename' (relative to self.root).

nilmdb/server/nilmdb.py

@@ -15,11 +15,9 @@ from nilmdb.utils.printf import *
 from nilmdb.server.interval import (Interval, DBInterval,
                                     IntervalSet, IntervalError)
 from nilmdb.server import bulkdata
-from nilmdb.server.errors import *
+from nilmdb.server.errors import NilmDBError, StreamError, OverlapError
 
 import sqlite3
-import time
-import sys
 import os
 import errno
 import bisect
@@ -80,7 +78,10 @@ class NilmDB(object):
     verbose = 0
 
     def __init__(self, basepath, sync=True, max_results=None,
-                 bulkdata_args={}):
+                 bulkdata_args=None):
+        if bulkdata_args is None:
+            bulkdata_args = {}
+
         # set up path
         self.basepath = os.path.abspath(basepath)
 
@@ -156,7 +157,7 @@ class NilmDB(object):
                 iset += DBInterval(start_time, end_time,
                                    start_time, end_time,
                                    start_pos, end_pos)
-        except IntervalError as e: # pragma: no cover
+        except IntervalError: # pragma: no cover
             raise NilmDBError("unexpected overlap in ranges table!")
 
         return iset

nilmdb/server/server.py

@@ -5,18 +5,16 @@
 from __future__ import absolute_import
 import nilmdb
 from nilmdb.utils.printf import *
-from nilmdb.server.errors import *
+from nilmdb.server.errors import NilmDBError
 
 import cherrypy
 import sys
-import time
 import os
 import simplejson as json
 import decorator
 import traceback
 
 try:
-    import cherrypy
     cherrypy.tools.json_out
 except: # pragma: no cover
     sys.stderr.write("Cherrypy 3.2+ required\n")
@@ -57,7 +55,7 @@ def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
     try:
         for val in func(*args, **kwargs):
             yield val
-    except (LookupError, UnicodeError) as e:
+    except (LookupError, UnicodeError):
         raise Exception("bug workaround; real exception is:\n" +
                         traceback.format_exc())
 
@@ -84,9 +82,9 @@ def exception_to_httperror(*expected):
 class Root(NilmApp):
     """Root application for NILM database"""
 
-    def __init__(self, db, version):
+    def __init__(self, db, sversion):
         super(Root, self).__init__(db)
-        self.server_version = version
+        self.server_version = sversion
 
     # /
     @cherrypy.expose
@@ -247,7 +245,7 @@ class Stream(NilmApp):
 
         # Now do the nilmdb insert, passing it the parser full of data.
         try:
-            result = self.db.stream_insert(path, start, end, parser.data)
+            self.db.stream_insert(path, start, end, parser.data)
         except NilmDBError as e:
             raise cherrypy.HTTPError("400 Bad Request", e.message)
 
@@ -309,7 +307,7 @@ class Stream(NilmApp):
         def content(start, end):
             # Note: disable chunked responses to see tracebacks from here.
             while True:
-                (intervals, restart) = self.db.stream_intervals(path,start,end)
+                (intervals, restart) = self.db.stream_intervals(path, start, end)
                 response = ''.join([ json.dumps(i) + "\n" for i in intervals ])
                 yield response
                 if restart == 0:
@@ -444,7 +442,7 @@ class Server(object):
             if not self.force_traceback:
                 if code >= 400 and code <= 499:
                     errordata["traceback"] = ""
-        except Exception as e: # pragma: no cover
+        except Exception: # pragma: no cover
             pass
         # Override the response type, which was previously set to text/html
         cherrypy.serving.response.headers['Content-Type'] = (

nilmdb/utils/diskusage.py

@@ -19,8 +19,8 @@ def du_bytes(path):
     """Like du -sb, returns total size of path in bytes."""
     size = os.path.getsize(path)
     if os.path.isdir(path):
-        for file in os.listdir(path):
-            filepath = os.path.join(path, file)
+        for thisfile in os.listdir(path):
+            filepath = os.path.join(path, thisfile)
             size += du_bytes(filepath)
     return size
 

nilmdb/utils/lrucache.py

@@ -5,7 +5,6 @@
 
 import collections
 import decorator
-import warnings
 
 def lru_cache(size = 10, onremove = None, keys = slice(None)):
     """Least-recently-used cache decorator.

nilmdb/utils/timer.py

@@ -2,7 +2,7 @@
 
 # Simple timer to time a block of code, for optimization debugging
 # use like:
-#   with nilmdb.Timer("flush"):
+#   with nilmdb.utils.Timer("flush"):
 #       foo.flush()
 
 from __future__ import print_function

nilmdb/utils/timestamper.py

@@ -3,19 +3,16 @@
 from nilmdb.utils.printf import *
 from nilmdb.utils import datetime_tz
 
-import time
-import os
-
 class Timestamper(object):
     """A file-like object that adds timestamps to lines of an input file."""
-    def __init__(self, file, ts_iter):
+    def __init__(self, infile, ts_iter):
         """file: filename, or another file-like object
            ts_iter: iterator that returns a timestamp string for
            each line of the file"""
-        if isinstance(file, basestring):
-            self.file = open(file, "r")
+        if isinstance(infile, basestring):
+            self.file = open(infile, "r")
         else:
-            self.file = file
+            self.file = infile
         self.ts_iter = ts_iter
 
     def close(self):
@@ -54,7 +51,7 @@ class Timestamper(object):
 
 class TimestamperRate(Timestamper):
     """Timestamper that uses a start time and a fixed rate"""
-    def __init__(self, file, start, rate, end = None):
+    def __init__(self, infile, start, rate, end = None):
         """
         file: file name or object
 
@@ -76,7 +73,7 @@ class TimestamperRate(Timestamper):
         # Handle case where we're passed a datetime or datetime_tz object
         if "totimestamp" in dir(start):
             start = start.totimestamp()
-        Timestamper.__init__(self, file, iterator(start, rate, end))
+        Timestamper.__init__(self, infile, iterator(start, rate, end))
         self.start = start
         self.rate = rate
     def __str__(self):
@@ -87,21 +84,21 @@ class TimestamperRate(Timestamper):
 
 class TimestamperNow(Timestamper):
     """Timestamper that uses current time"""
-    def __init__(self, file):
+    def __init__(self, infile):
         def iterator():
             while True:
                 now = datetime_tz.datetime_tz.utcnow().totimestamp()
                 yield sprintf("%.6f ", now)
-        Timestamper.__init__(self, file, iterator())
+        Timestamper.__init__(self, infile, iterator())
     def __str__(self):
         return "TimestamperNow(...)"
 
 class TimestamperNull(Timestamper):
     """Timestamper that adds nothing to each line"""
-    def __init__(self, file):
+    def __init__(self, infile):
         def iterator():
             while True:
                 yield ""
-        Timestamper.__init__(self, file, iterator())
+        Timestamper.__init__(self, infile, iterator())
     def __str__(self):
         return "TimestamperNull(...)"

setup.py

@@ -35,6 +35,7 @@ setup(name='nilmdb',
                         ],
       install_requires = [ 'distribute',
                            'decorator',
+                           'cherrypy >= 3.2',
                            ],
       packages = [ 'nilmdb',
                    'nilmdb.utils',