Browse Source

Clean up some pylint warnings and potential errors

tags/nilmdb-1.0
Jim Paris 9 years ago
parent
commit
750d9e3c38
21 changed files with 304 additions and 77 deletions
  1. +250
    -0
      .pylintrc
  2. +1
    -1
      Makefile
  3. +2
    -2
      nilmdb/__init__.py
  4. +1
    -1
      nilmdb/client/__init__.py
  5. +3
    -6
      nilmdb/client/client.py
  6. +1
    -6
      nilmdb/client/httpclient.py
  7. +0
    -1
      nilmdb/cmdline/cmdline.py
  8. +0
    -2
      nilmdb/cmdline/create.py
  9. +0
    -2
      nilmdb/cmdline/extract.py
  10. +1
    -3
      nilmdb/cmdline/insert.py
  11. +0
    -2
      nilmdb/cmdline/list.py
  12. +0
    -1
      nilmdb/cmdline/remove.py
  13. +5
    -3
      nilmdb/server/__init__.py
  14. +13
    -16
      nilmdb/server/bulkdata.py
  15. +6
    -5
      nilmdb/server/nilmdb.py
  16. +7
    -9
      nilmdb/server/server.py
  17. +2
    -2
      nilmdb/utils/diskusage.py
  18. +0
    -1
      nilmdb/utils/lrucache.py
  19. +1
    -1
      nilmdb/utils/timer.py
  20. +10
    -13
      nilmdb/utils/timestamper.py
  21. +1
    -0
      setup.py

+ 250
- 0
.pylintrc View File

@@ -0,0 +1,250 @@
# -*- conf -*-
[MASTER]

# Specify a configuration file.
#rcfile=

# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=

# Profiled execution.
profile=no

# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=datetime_tz

# Pickle collected data for later comparisons.
persistent=no

# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=


[MESSAGES CONTROL]

# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time.
#enable=

# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once).
disable=C0111,R0903,R0201,R0914,R0912,W0142,W0703,W0702


[REPORTS]

# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html
output-format=parseable

# Include message's id in output
include-ids=yes

# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no

# Tells whether to display a full report or only the messages
reports=yes

# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)

# Add a comment according to your evaluation note. This is used by the global
# evaluation report (RP0004).
comment=no


[SIMILARITIES]

# Minimum lines number of a similarity.
min-similarity-lines=4

# Ignore comments when computing similarities.
ignore-comments=yes

# Ignore docstrings when computing similarities.
ignore-docstrings=yes


[TYPECHECK]

# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes

# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set).
ignored-classes=SQLObject

# When zope mode is activated, add a predefined set of Zope acquired attributes
# to generated-members.
zope=no

# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E0201 when accessed. Python regular
# expressions are accepted.
generated-members=REQUEST,acl_users,aq_parent


[FORMAT]

# Maximum number of characters on a single line.
max-line-length=80

# Maximum number of lines in a module
max-module-lines=1000

# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '


[MISCELLANEOUS]

# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO


[VARIABLES]

# Tells whether we should check for unused import in __init__ files.
init-import=no

# A regular expression matching the beginning of the name of dummy variables
# (i.e. not used).
dummy-variables-rgx=_|dummy

# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=


[BASIC]

# Required attributes for module, separated by a comma
required-attributes=

# List of builtins function names that should not be used, separated by a comma
bad-functions=apply,input

# Regular expression which should only match correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

# Regular expression which should only match correct module level names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|version)$

# Regular expression which should only match correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$

# Regular expression which should only match correct function names
function-rgx=[a-z_][a-z0-9_]{0,30}$

# Regular expression which should only match correct method names
method-rgx=[a-z_][a-z0-9_]{0,30}$

# Regular expression which should only match correct instance attribute names
attr-rgx=[a-z_][a-z0-9_]{0,30}$

# Regular expression which should only match correct argument names
argument-rgx=[a-z_][a-z0-9_]{0,30}$

# Regular expression which should only match correct variable names
variable-rgx=[a-z_][a-z0-9_]{0,30}$

# Regular expression which should only match correct list comprehension /
# generator expression variable names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$

# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_

# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata

# Regular expression which should only match functions or classes name which do
# not require a docstring
no-docstring-rgx=__.*__


[CLASSES]

# List of interface methods to ignore, separated by a comma. This is used for
# instance to not check methods defines in Zope's Interface base class.
ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by

# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp

# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls


[DESIGN]

# Maximum number of arguments for function / method
max-args=5

# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*

# Maximum number of locals for function / method body
max-locals=15

# Maximum number of return / yield for function / method body
max-returns=6

# Maximum number of branch for function / method body
max-branchs=12

# Maximum number of statements in function / method body
max-statements=50

# Maximum number of parents for a class (see R0901).
max-parents=7

# Maximum number of attributes for a class (see R0902).
max-attributes=7

# Minimum number of public methods for a class (see R0903).
min-public-methods=2

# Maximum number of public methods for a class (see R0904).
max-public-methods=20


[IMPORTS]

# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,string,TERMIOS,Bastion,rexec

# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=

# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=

# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=


[EXCEPTIONS]

# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception

+ 1
- 1
Makefile View File

@@ -1,7 +1,7 @@
all: test

lint:
pylint -f parseable nilmdb
pylint --rcfile=.pylintrc nilmdb

test:
python runtests.py


+ 2
- 2
nilmdb/__init__.py View File

@@ -1,4 +1,4 @@
"""Main NilmDB import"""

from server import NilmDB, Server
from client import Client
from nilmdb.server import NilmDB, Server
from nilmdb.client import Client

+ 1
- 1
nilmdb/client/__init__.py View File

@@ -1,4 +1,4 @@
"""nilmdb.client"""

from .client import Client
from .errors import *
from .errors import ClientError, ServerError, Error

+ 3
- 6
nilmdb/client/client.py View File

@@ -5,19 +5,14 @@
import nilmdb
import nilmdb.utils
import nilmdb.client.httpclient
from nilmdb.utils.printf import *

import time
import sys
import re
import os
import simplejson as json
import itertools

version = "1.0"

def float_to_string(f):
# Use repr to maintain full precision in the string output.
"""Use repr to maintain full precision in the string output."""
return repr(float(f))

class Client(object):
@@ -151,6 +146,8 @@ class Client(object):
block_data = ""
block_start = start
result = None
line = None
nextline = None
for (line, nextline) in nilmdb.utils.misc.pairwise(data):
# If we don't have a starting time, extract it from the first line
if block_start is None:


+ 1
- 6
nilmdb/client/httpclient.py View File

@@ -2,13 +2,8 @@

import nilmdb
import nilmdb.utils
from nilmdb.utils.printf import *
from nilmdb.client.errors import *
from nilmdb.client.errors import ClientError, ServerError, Error

import time
import sys
import re
import os
import simplejson as json
import urlparse
import pycurl


+ 0
- 1
nilmdb/cmdline/cmdline.py View File

@@ -4,7 +4,6 @@ import nilmdb
from nilmdb.utils.printf import *
from nilmdb.utils import datetime_tz

import dateutil.parser
import sys
import re
import argparse


+ 0
- 2
nilmdb/cmdline/create.py View File

@@ -1,9 +1,7 @@
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client
import textwrap

from argparse import ArgumentDefaultsHelpFormatter as def_form
from argparse import RawDescriptionHelpFormatter as raw_form

def setup(self, sub):


+ 0
- 2
nilmdb/cmdline/extract.py View File

@@ -1,8 +1,6 @@
from __future__ import print_function
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client
import sys

def setup(self, sub):
cmd = sub.add_parser("extract", help="Extract data",


+ 1
- 3
nilmdb/cmdline/insert.py View File

@@ -53,8 +53,6 @@ def cmd_insert(self):
if len(streams) != 1:
self.die("error getting stream info for path %s", self.args.path)

layout = streams[0][1]

if self.args.start and len(self.args.file) != 1:
self.die("error: --start can only be used with one input file")

@@ -93,7 +91,7 @@ def cmd_insert(self):

# Insert the data
try:
result = self.client.stream_insert(self.args.path, ts)
self.client.stream_insert(self.args.path, ts)
except nilmdb.client.Error as e:
# TODO: It would be nice to be able to offer better errors
# here, particularly in the case of overlap, which just shows


+ 0
- 2
nilmdb/cmdline/list.py View File

@@ -1,6 +1,4 @@
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client

import fnmatch
import argparse


+ 0
- 1
nilmdb/cmdline/remove.py View File

@@ -1,7 +1,6 @@
from nilmdb.utils.printf import *
import nilmdb
import nilmdb.client
import sys

def setup(self, sub):
cmd = sub.add_parser("remove", help="Remove data",


+ 5
- 3
nilmdb/server/__init__.py View File

@@ -1,5 +1,7 @@
"""nilmdb.server"""

from __future__ import absolute_import

# Try to set up pyximport to automatically rebuild Cython modules. If
# this doesn't work, it's OK, as long as the modules were built externally.
# (e.g. python setup.py build_ext --inplace)
@@ -10,6 +12,6 @@ try:
except: # pragma: no cover
pass

from .nilmdb import NilmDB
from .server import Server
from .errors import *
from nilmdb.server.nilmdb import NilmDB
from nilmdb.server.server import Server
from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

+ 13
- 16
nilmdb/server/bulkdata.py View File

@@ -8,10 +8,8 @@ import nilmdb
from nilmdb.utils.printf import *

import os
import sys
import cPickle as pickle
import struct
import fnmatch
import mmap
import re

@@ -91,8 +89,7 @@ class BulkData(object):
"float32": 'f',
"float64": 'd',
}
for n in range(layout.count):
struct_fmt += struct_mapping[layout.datatype]
struct_fmt += struct_mapping[layout.datatype] * layout.count
except KeyError:
raise ValueError("no such layout, or bad data types")

@@ -185,12 +182,12 @@ class Table(object):
packer = struct.Struct(struct_fmt)
rows_per_file = max(file_size // packer.size, 1)

format = { "rows_per_file": rows_per_file,
"files_per_dir": files_per_dir,
"struct_fmt": struct_fmt,
"version": 1 }
fmt = { "rows_per_file": rows_per_file,
"files_per_dir": files_per_dir,
"struct_fmt": struct_fmt,
"version": 1 }
with open(os.path.join(root, "_format"), "wb") as f:
pickle.dump(format, f, 2)
pickle.dump(fmt, f, 2)

# Normal methods
def __init__(self, root):
@@ -199,15 +196,15 @@ class Table(object):

# Load the format and build packer
with open(os.path.join(self.root, "_format"), "rb") as f:
format = pickle.load(f)
fmt = pickle.load(f)

if format["version"] != 1: # pragma: no cover (just future proofing)
raise NotImplementedError("version " + format["version"] +
if fmt["version"] != 1: # pragma: no cover (just future proofing)
raise NotImplementedError("version " + fmt["version"] +
" bulk data store not supported")

self.rows_per_file = format["rows_per_file"]
self.files_per_dir = format["files_per_dir"]
self.packer = struct.Struct(format["struct_fmt"])
self.rows_per_file = fmt["rows_per_file"]
self.files_per_dir = fmt["files_per_dir"]
self.packer = struct.Struct(fmt["struct_fmt"])
self.file_size = self.packer.size * self.rows_per_file

# Find nrows
@@ -278,7 +275,7 @@ class Table(object):

# Cache open files
@nilmdb.utils.lru_cache(size = fd_cache_size,
keys = slice(0,3), # exclude newsize
keys = slice(0, 3), # exclude newsize
onremove = lambda x: x.close())
def mmap_open(self, subdir, filename, newsize = None):
"""Open and map a given 'subdir/filename' (relative to self.root).


+ 6
- 5
nilmdb/server/nilmdb.py View File

@@ -15,11 +15,9 @@ from nilmdb.utils.printf import *
from nilmdb.server.interval import (Interval, DBInterval,
IntervalSet, IntervalError)
from nilmdb.server import bulkdata
from nilmdb.server.errors import *
from nilmdb.server.errors import NilmDBError, StreamError, OverlapError

import sqlite3
import time
import sys
import os
import errno
import bisect
@@ -80,7 +78,10 @@ class NilmDB(object):
verbose = 0

def __init__(self, basepath, sync=True, max_results=None,
bulkdata_args={}):
bulkdata_args=None):
if bulkdata_args is None:
bulkdata_args = {}

# set up path
self.basepath = os.path.abspath(basepath)

@@ -156,7 +157,7 @@ class NilmDB(object):
iset += DBInterval(start_time, end_time,
start_time, end_time,
start_pos, end_pos)
except IntervalError as e: # pragma: no cover
except IntervalError: # pragma: no cover
raise NilmDBError("unexpected overlap in ranges table!")

return iset


+ 7
- 9
nilmdb/server/server.py View File

@@ -5,18 +5,16 @@
from __future__ import absolute_import
import nilmdb
from nilmdb.utils.printf import *
from nilmdb.server.errors import *
from nilmdb.server.errors import NilmDBError

import cherrypy
import sys
import time
import os
import simplejson as json
import decorator
import traceback

try:
import cherrypy
cherrypy.tools.json_out
except: # pragma: no cover
sys.stderr.write("Cherrypy 3.2+ required\n")
@@ -57,7 +55,7 @@ def workaround_cp_bug_1200(func, *args, **kwargs): # pragma: no cover
try:
for val in func(*args, **kwargs):
yield val
except (LookupError, UnicodeError) as e:
except (LookupError, UnicodeError):
raise Exception("bug workaround; real exception is:\n" +
traceback.format_exc())

@@ -84,9 +82,9 @@ def exception_to_httperror(*expected):
class Root(NilmApp):
"""Root application for NILM database"""

def __init__(self, db, version):
def __init__(self, db, sversion):
super(Root, self).__init__(db)
self.server_version = version
self.server_version = sversion

# /
@cherrypy.expose
@@ -247,7 +245,7 @@ class Stream(NilmApp):

# Now do the nilmdb insert, passing it the parser full of data.
try:
result = self.db.stream_insert(path, start, end, parser.data)
self.db.stream_insert(path, start, end, parser.data)
except NilmDBError as e:
raise cherrypy.HTTPError("400 Bad Request", e.message)

@@ -309,7 +307,7 @@ class Stream(NilmApp):
def content(start, end):
# Note: disable chunked responses to see tracebacks from here.
while True:
(intervals, restart) = self.db.stream_intervals(path,start,end)
(intervals, restart) = self.db.stream_intervals(path, start, end)
response = ''.join([ json.dumps(i) + "\n" for i in intervals ])
yield response
if restart == 0:
@@ -444,7 +442,7 @@ class Server(object):
if not self.force_traceback:
if code >= 400 and code <= 499:
errordata["traceback"] = ""
except Exception as e: # pragma: no cover
except Exception: # pragma: no cover
pass
# Override the response type, which was previously set to text/html
cherrypy.serving.response.headers['Content-Type'] = (


+ 2
- 2
nilmdb/utils/diskusage.py View File

@@ -19,8 +19,8 @@ def du_bytes(path):
"""Like du -sb, returns total size of path in bytes."""
size = os.path.getsize(path)
if os.path.isdir(path):
for file in os.listdir(path):
filepath = os.path.join(path, file)
for thisfile in os.listdir(path):
filepath = os.path.join(path, thisfile)
size += du_bytes(filepath)
return size



+ 0
- 1
nilmdb/utils/lrucache.py View File

@@ -5,7 +5,6 @@

import collections
import decorator
import warnings

def lru_cache(size = 10, onremove = None, keys = slice(None)):
"""Least-recently-used cache decorator.


+ 1
- 1
nilmdb/utils/timer.py View File

@@ -2,7 +2,7 @@

# Simple timer to time a block of code, for optimization debugging
# use like:
# with nilmdb.Timer("flush"):
# with nilmdb.utils.Timer("flush"):
# foo.flush()

from __future__ import print_function


+ 10
- 13
nilmdb/utils/timestamper.py View File

@@ -3,19 +3,16 @@
from nilmdb.utils.printf import *
from nilmdb.utils import datetime_tz

import time
import os

class Timestamper(object):
"""A file-like object that adds timestamps to lines of an input file."""
def __init__(self, file, ts_iter):
def __init__(self, infile, ts_iter):
"""file: filename, or another file-like object
ts_iter: iterator that returns a timestamp string for
each line of the file"""
if isinstance(file, basestring):
self.file = open(file, "r")
if isinstance(infile, basestring):
self.file = open(infile, "r")
else:
self.file = file
self.file = infile
self.ts_iter = ts_iter

def close(self):
@@ -54,7 +51,7 @@ class Timestamper(object):

class TimestamperRate(Timestamper):
"""Timestamper that uses a start time and a fixed rate"""
def __init__(self, file, start, rate, end = None):
def __init__(self, infile, start, rate, end = None):
"""
file: file name or object

@@ -76,7 +73,7 @@ class TimestamperRate(Timestamper):
# Handle case where we're passed a datetime or datetime_tz object
if "totimestamp" in dir(start):
start = start.totimestamp()
Timestamper.__init__(self, file, iterator(start, rate, end))
Timestamper.__init__(self, infile, iterator(start, rate, end))
self.start = start
self.rate = rate
def __str__(self):
@@ -87,21 +84,21 @@ class TimestamperRate(Timestamper):

class TimestamperNow(Timestamper):
"""Timestamper that uses current time"""
def __init__(self, file):
def __init__(self, infile):
def iterator():
while True:
now = datetime_tz.datetime_tz.utcnow().totimestamp()
yield sprintf("%.6f ", now)
Timestamper.__init__(self, file, iterator())
Timestamper.__init__(self, infile, iterator())
def __str__(self):
return "TimestamperNow(...)"

class TimestamperNull(Timestamper):
"""Timestamper that adds nothing to each line"""
def __init__(self, file):
def __init__(self, infile):
def iterator():
while True:
yield ""
Timestamper.__init__(self, file, iterator())
Timestamper.__init__(self, infile, iterator())
def __str__(self):
return "TimestamperNull(...)"

+ 1
- 0
setup.py View File

@@ -35,6 +35,7 @@ setup(name='nilmdb',
],
install_requires = [ 'distribute',
'decorator',
'cherrypy >= 3.2',
],
packages = [ 'nilmdb',
'nilmdb.utils',


Loading…
Cancel
Save