Rework Cython and external module support.

Now we build Cython modules only if cython >= 0.16 is present.
Tarballs made by "make sdist" include the Cython-generated *.c files,
and so Cython isn't required on the end user machine at all.

Makefile

@@ -24,7 +24,7 @@ clean::
 	find . -name '*pyc' | xargs rm -f
 	rm -f .coverage
 	rm -rf tests/*testdb*
-	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.c nilmdb/server/*.so MANIFEST.in
+	rm -rf nilmdb.egg-info/ build/ nilmdb/server/*.so MANIFEST.in
 	make -C docs clean
 
 .PHONY: all build dist sdist install docs lint test clean

README.txt

@@ -3,7 +3,14 @@ by Jim Paris <jim@jtan.com>
 
 Prerequisites:
 
-  sudo apt-get install python2.7 python-setuptools cython python-cherrypy3 python-decorator python-nose python-coverage
+  # Runtime and build environments
+  sudo apt-get install python2.7 python2.7-dev python-setuptools cython
+
+  # Base NilmDB dependencies
+  sudo apt-get install python-cherrypy3 python-decorator python-simplejson python-pycurl python-dateutil python-tz
+
+  # Tools for running tests
+  sudo apt-get install python-nose python-coverage
 
 Install:
 

nilmdb/server/__init__.py

@@ -6,9 +6,14 @@ from __future__ import absolute_import
 # this doesn't work, it's OK, as long as the modules were built externally.
 # (e.g. python setup.py build_ext --inplace)
 try:
+    import Cython
+    import distutils.version
+    if (distutils.version.LooseVersion(Cython.__version__) <
+        distutils.version.LooseVersion("0.16")): # pragma: no cover
+        raise ImportError("Cython version too old")
     import pyximport
-    pyximport.install()
-except: # pragma: no cover
+    pyximport.install(inplace = True, build_in_temp = False)
+except ImportError: # pragma: no cover
     pass
 
 import nilmdb.server.layout

setup.py

@@ -11,11 +11,12 @@
 
 import traceback
 import sys
+import os
 
 try:
     from setuptools import setup, find_packages
     from distutils.extension import Extension
-    from Cython.Build import cythonize
+    import distutils.version
 except ImportError:
     traceback.print_exc()
     print "Please install the prerequisites listed in README.txt"
@@ -26,15 +27,32 @@ except ImportError:
 try: import multiprocessing
 except: pass
 
-if len(sys.argv) == 2 and sys.argv[1] == 'sdist':
-    # When building just a source distribution package, skip Cython,
-    # so that the generated .c files don't get included.  This isn't
-    # foolproof, but the worst case is that the .c files get included,
-    # which is fine.
-    cython_modules = []
-else:
-    # Build cython modules.
-    cython_modules = cythonize("**/*.pyx")
+# Use Cython if it's new enough, otherwise use preexisting C files.
+cython_modules = [ 'nilmdb.server.interval',
+                   'nilmdb.server.layout',
+                   'nilmdb.server.rbtree' ]
+try:
+    import Cython
+    from Cython.Build import cythonize
+    if (distutils.version.LooseVersion(Cython.__version__) <
+        distutils.version.LooseVersion("0.16")):
+        print "Cython version", Cython.__version__, "is too old; not using it."
+        raise ImportError()
+    use_cython = True
+except ImportError:
+    use_cython = False
+
+ext_modules = []
+for modulename in cython_modules:
+    filename = modulename.replace('.','/')
+    if use_cython:
+        ext_modules.extend(cythonize(filename + ".pyx"))
+    else:
+        cfile = filename + ".c"
+        if not os.path.exists(cfile):
+            raise Exception("Missing source file " + cfile + ".  "
+                            "Try installing cython >= 0.16.")
+        ext_modules.append(Extension(modulename, [ cfile ]))
 
 # We need a MANIFEST.in.  Generate it here rather than polluting the
 # repository with yet another setup-related file.
@@ -72,10 +90,9 @@ setup(name='nilmdb',
       tests_require = [ 'nose',
                         'coverage',
                         ],
-      setup_requires = [ 'cython',
-                        ],
-      install_requires = [ 'distribute',
-                           'decorator',
+      setup_requires = [ 'distribute',
+                         ],
+      install_requires = [ 'decorator',
                            'cherrypy >= 3.2',
                            ],
       packages = [ 'nilmdb',
@@ -92,6 +109,6 @@ setup(name='nilmdb',
               'nilmdb-server = nilmdb.scripts.nilmdb_server:main',
               ],
           },
-      ext_modules = cython_modules,
+      ext_modules = ext_modules,
       zip_safe = False,
       )