commit 918765b14d9de3dd5abe14e43216e43fe71d193d
Author: Jim Paris <jim@jtan.com>
Date:   Sat Aug 17 22:28:49 2013 -0400

    Initial commit

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d39cddd
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+fs
+iso
+mnt
+*.iso
diff --git a/README b/README
new file mode 100644
index 0000000..dea96bf
--- /dev/null
+++ b/README
@@ -0,0 +1,30 @@
+This live CD is customized by roughly following the guidelines at:
+https://
+  help.ubuntu.com/community/LiveCDCustomization
+  help.ubuntu.com/community/LiveCDCustomizationFromScratch
+  help.ubuntu.com/community/MakeALiveCD/DVD/BootableFlashFromHarddiskInstall
+plus a lot of other info, like ubiquity and casper source code, and
+additions for making the resulting iso EFI bootable.
+
+Run things in generally this order:
+
+extractiso.sh
+  extract the original iso
+
+customize.sh
+  apply customizations
+
+enter.sh
+  chroot into the install (for optional manual customization)
+
+buildiso.sh
+  build the new iso
+
+run.sh -c
+  boot the iso in qemu with a blank hdd, to test the install
+
+run.sh -d
+  boot the installed hdd in qemu
+
+cleanup.sh
+  remove everything but the isos
diff --git a/buildiso.sh b/buildiso.sh
new file mode 100755
index 0000000..ba97945
--- /dev/null
+++ b/buildiso.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# build the iso
+. config || exit 0
+
+set -x
+set -e
+
+# copy kernel if changed
+if [ ${FS}/initrd.img -nt ${ISO}/casper/initrd.lz ] ; then
+    sudo cp ${FS}/vmlinuz ${ISO}/casper/vmlinuz
+    sudo sh -c "zcat ${FS}/initrd.img | lzma > ${ISO}/casper/initrd.lz"
+fi
+
+# manifests
+sudo chmod +w ${ISO}/casper/filesystem.manifest
+sudo chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
+    | sudo tee ${ISO}/casper/filesystem.manifest >/dev/null
+
+# squashfs
+sudo rm -f ${ISO}/casper/filesystem.squashfs
+sudo mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
+printf $(sudo du -sx --block-size=1 ${FS} | cut -f1) \
+    | sudo tee ${ISO}/casper/filesystem.size
+
+# md5sums
+cd ${ISO}
+sudo rm -f md5sum.txt
+sudo find . -type f -print0 \
+    | sudo xargs -0 md5sum \
+    | grep -v isolinux/boot.cat \
+    | sudo tee md5sum.txt >/dev/null
+cd ..
+
+# build CD
+cd ${ISO}
+REALOUT=$(realpath ${OUTPUT})
+sudo xorriso -as mkisofs \
+    -D -r -V "NilmDBuntu" -cache-inodes -J -l \
+    -input-charset utf-8 -o ${REALOUT} \
+    -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
+    -boot-load-size 4 -boot-info-table \
+    -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
+    .
+cd ..
+sudo chown jim:jim ${REALOUT}
diff --git a/cleanup.sh b/cleanup.sh
new file mode 100755
index 0000000..eec26b8
--- /dev/null
+++ b/cleanup.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# build the iso
+. config || exit 0
+
+sudo umount mnt || ture
+sudo rm --one-file-system -rf ${FS} ${ISO} ${MNT} ${DISK}
diff --git a/config b/config
new file mode 100644
index 0000000..8eb7616
--- /dev/null
+++ b/config
@@ -0,0 +1,16 @@
+# output image
+VERSION="12.10.0"
+OUTPUT="nilmdbuntu-${VERSION}.iso"
+
+# original ISOs
+ORIG="xubuntu-12.10-desktop-amd64.iso"
+ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/12.10/release/${ORIG}"
+
+# work directories, temporary qemu HD image
+FS="fs"
+ISO="iso"
+MNT="mnt"
+DISK="disk.img"
+
+# local user
+USER=jim:jim
diff --git a/customize-inner.sh b/customize-inner.sh
new file mode 100644
index 0000000..2b03189
--- /dev/null
+++ b/customize-inner.sh
@@ -0,0 +1,180 @@
+#!/bin/bash
+
+if [ "$IN_CHROOT" != "1" ] ; then
+    echo This is supposed to run inside the chroot, oops
+    exit 1
+fi
+
+set -e
+set -x
+
+# Set up live username and hostname
+cat >/etc/casper.conf <<"EOF"
+export USERNAME="ubuntu"
+export USERFULLNAME="Live session user"
+export HOST="nilmdb"
+export BUILD_SYSTEM="Ubuntu"
+export FLAVOUR="NilmDBuntu"
+EOF
+
+# Upgrade packages, remove old kernels
+apt-get update
+apt-get -y dist-upgrade
+apt-get -y --purge autoremove
+for VER in $(ls --sort=version /lib/modules/ | head -n -1) ; do
+    apt-get -y --purge remove ".*$VER.*"
+done
+
+# install nilmdb things
+apt-get -y install \
+    python2.7 \
+    python2.7-dev \
+    python-setuptools \
+    python-pip \
+    cython \
+    git \
+    build-essential \
+    python-cherrypy3 \
+    python-decorator \
+    python-simplejson \
+    python-requests \
+    python-dateutil \
+    python-tz \
+    python-progressbar \
+    python-psutil \
+    python-numpy \
+    python-nose \
+    python-coverage \
+    apache2 \
+    libapache2-mod-wsgi \
+    python-scipy \
+    python-daemon
+
+# install other useful utilities
+apt-get -y install \
+    emacs-goodies-el \
+    emacs23-nox \
+    octave \
+    octave-signal \
+    octave-missing-functions \
+    gnuplot \
+    curl \
+    gddrescue \
+    help2man \
+    luatex \
+    pgf \
+    moreutils \
+    ntfsprogs \
+    subversion \
+    dlocate \
+    ack-grep
+
+# Set up & install postfix for local mail delivery
+debconf-set-selections <<"EOF"
+postfix postfix/mailname string "localdomain"
+postfix postfix/main_mailer_type select "Local only"
+EOF
+apt-get -y install postfix
+
+# Create nilmdb user to run the database
+adduser --system --group --shell /bin/bash --disabled-password nilmdb
+
+# Create WSGI scripts
+cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
+import nilmdb.server
+application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
+EOF
+cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
+import nilmrun.server
+application = nilmrun.server.wsgi_application("/nilmrun")
+EOF
+
+# Create apache config by hacking up the default one.  Might be a better way
+# to do this, and it'll probably break on apache 2.4, but...
+DEF=/etc/apache2/sites-available/default
+perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig
+perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF
+cat >>$DEF <<"EOF"
+	## NilmDB start
+	WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
+	WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
+	<Location /nilmdb>
+	    WSGIProcessGroup nilmdb-procgroup
+	    WSGIApplicationGroup nilmdb-appgroup
+	</Location>
+
+	WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
+	WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
+	<Location /nilmrun>
+	    WSGIProcessGroup nilmrun-procgroup
+	    WSGIApplicationGroup nilmrun-appgroup
+	</Location>
+        ## NilmDB end
+EOF
+perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF
+
+# Create nilmdb capture, processing, and cleanup files
+cat > /home/nilmdb/capture.sh <<"EOF"
+#!/bin/bash -e
+
+# Don't run capture if we're running off a live CD
+if grep -q boot=casper /proc/cmdline ; then
+    exit 0
+fi
+
+nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
+    "ethstream -a 192.168.1.209 -n 6 -r 8000" \
+    "nilm-insert -m 10 -r 8000 --live /data/raw"
+EOF
+cat > /home/nilmdb/process.sh <<"EOF"
+#!/bin/bash -e
+
+# Ensure only one copy of this code runs at a time:
+LOCKFILE="/tmp/nilmdb-process.lock"
+exec 99>"$LOCKFILE"
+flock -n -x 99 || exit 0
+trap 'rm -f "$LOCKFILE"' 0
+
+nilm-sinefit -c 4 /data/raw /data/sinefit
+nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
+nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
+nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
+nilm-decimate-auto /data/raw "/data/prep*"
+nilm-cleanup --yes /home/nilmdb/cleanup.cfg
+EOF
+cat > /home/nilmdb/cleanup.cfg <<"EOF"
+[/data/prep-*]
+keep = 1y
+
+[/data/raw]
+keep = 2w
+
+[/data/sinefit]
+keep = 1y
+decimated = false
+EOF
+
+# Fix permissions
+chown -R nilmdb:nilmdb /home/nilmdb
+chmod +x /home/nilmdb/{capture,process}.sh
+
+# Fetch and build everything.  Put it in the nilmdb dir
+echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
+GIT=https://git.jim.sh/jim/lees
+rm -rf /home/nilmdb/git
+mkdir /home/nilmdb/git
+chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
+REPOS="nilmdb nilmtools nilmrun ethstream"
+
+# check it out as nilmdb, so the .netrc gets used
+for repo in $REPOS; do
+    sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
+done
+
+# build as root, because we need to do that for the install
+for repo in $REPOS; do
+    make -C /home/nilmdb/git/$repo install
+done
+
+# fix up all permissions in git dir, so nilmdb user can play with it later
+chown -R nilmdb:nilmdb /home/nilmdb/git
diff --git a/customize.sh b/customize.sh
new file mode 100755
index 0000000..5951cf8
--- /dev/null
+++ b/customize.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+. config || exit 0
+set -e
+set -x
+
+# Customize the outer ISO image
+perl -p -i -e \
+    "s/DISKNAME.*/DISKNAME  NilmDBuntu $VERSION by Jim Paris/" \
+    ${ISO}/README.diskdefines
+
+# The .disk/info file is important -- it's used by ubiquity to extract
+# out the distro name in dialogs, and I think casper might use it too
+echo "NilmDBuntu $VERSION by Jim Paris" > ${ISO}/.disk/info
+
+# Include openssh server at install time.  Can't include it in the
+# image, because then we'd share keys on all machines, I think.
+cp ${ISO}/preseed/xubuntu.seed ${ISO}/preseed/nilmdbuntu.seed
+cat >> ${ISO}/preseed/nilmdbuntu.seed <<EOF
+d-i pkgsel/include string openssh-server
+EOF
+
+# Rename Xubuntu to NilmDBuntu in boot scripts; also changes
+# preseed file to the custom one
+perl -p -i \
+    -e "s/Xubuntu/NilmDBuntu/g;" \
+    -e "s/xubuntu/nilmdbuntu/g;" \
+    ${ISO}/isolinux/txt.cfg ${ISO}/boot/grub/{grub,loopback}.cfg
+
+# Remove quiet and splash from boot command lines.  Easier than a
+# custom plymouth theme and it's helpful.
+perl -p -i -e "s/ quiet splash//g;" \
+    ${ISO}/isolinux/txt.cfg ${ISO}/boot/grub/{grub,loopback}.cfg
+
+# Run the customize-inner.sh script inside the chroot
+sudo cp customize-inner.sh ${FS}/root/customize-inner.sh
+sudo chmod +x ${FS}/root/customize-inner.sh
+./enter.sh "cd /root ; ./customize-inner.sh"
diff --git a/enter.sh b/enter.sh
new file mode 100755
index 0000000..b6fa263
--- /dev/null
+++ b/enter.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# make sure this was run as root
+if [ $UID -ne 0 ] ; then
+    echo "Need to be root; trying sudo"
+    exec sudo $0 "$@"
+fi
+
+# enter the chroot and run the command (if supplied) or a shell
+. config || exit 0
+
+run() {
+    echo "+" "$1"
+    chroot ${FS} env -i \
+        HOME=/root \
+        PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+        TERM=$TERM \
+        IN_CHROOT=1 \
+        bash -c "$1"
+    RET=$?
+    if [ $RET -ne 0 ] && [ "$1" != "exec bash" ] ; then
+        printf "%s\n" "----------- WARNING: failed with exit code $RET"
+        sleep 5
+    fi
+}
+
+set -e
+mount -t proc none ${FS}/proc
+mount -t sysfs none ${FS}/sys
+mount -t devpts none ${FS}/dev/pts
+
+run "echo 'nameserver 8.8.8.8' > /etc/resolv.conf"
+run "dbus-uuidgen > /var/lib/dbus/machine-id"
+run "dpkg-divert --local --rename --add /sbin/initctl"
+run "ln -s /bin/true /sbin/initctl"
+run "dpkg-divert --local --rename --add /usr/sbin/update-grub"
+run "ln -s /bin/true /usr/sbin/update-grub"
+
+set +e
+if [ -z "$1" ] ; then
+    run "exec bash"
+else
+    run "$1"
+fi
+run "apt-get clean"
+run "rm /sbin/initctl"
+run "dpkg-divert --rename --remove /sbin/initctl"
+run "rm /usr/sbin/update-grub"
+run "dpkg-divert --rename --remove /usr/sbin/update-grub"
+run "rm /var/lib/dbus/machine-id"
+run "> /etc/resolv.conf"
+run "rm -rf /tmp/* /tmp/.??* /root/.bash_history"
+
+umount ${FS}/dev/pts
+umount ${FS}/sys/kernel/security || true
+umount ${FS}/sys
+umount ${FS}/proc
+
+echo "cleaned up"
diff --git a/extractiso.sh b/extractiso.sh
new file mode 100755
index 0000000..3c860aa
--- /dev/null
+++ b/extractiso.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# build the iso
+. config || exit 0
+
+if [ "$1" != "ok" ]; then
+    if [ -e ${ISO} ] || [ -e ${FS} ]; then
+        echo "remove \"${ISO}\" and \"${FS}\" dirs first,"
+        echo "or pass \"ok\" as an argument to remove them"
+        exit 0
+    fi
+fi
+
+set -x
+set -e
+
+# download it if it doesn't exist
+if ! [ -e ${ORIG} ] ; then
+    wget -O "${ORIG}" "${ORIGURL}"
+fi
+
+# mount it
+sudo umount ${MNT} || true
+sudo rm -rf ${MNT} ${ISO} ${FS}
+sudo mkdir ${MNT}
+sudo mount -o loop,ro "$ORIG" ${MNT}
+
+# copy data
+sudo mkdir ${ISO}
+sudo rsync --exclude=/casper/filesystem.squashfs -a ${MNT}/ ${ISO}
+sudo umount ${MNT}
+sudo chown -R ${USER} ${ISO}
+chmod -R u+w ${ISO}
+
+# copy squashfs
+sudo unsquashfs -d ${FS} ${MNT}/casper/filesystem.squashfs
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..759ac8c
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# try to boot the ISO in qemu
+. config || exit 0
+
+usage() {
+    cat <<EOF
+usage: $0 [-i iso] <-c | -d>
+
+  -c   Make an empty disk image and boot from CD
+  -d   Boot the disk image
+  -i   Path to iso
+EOF
+    exit 0
+}
+
+boot=""
+iso="${OUTPUT}"
+
+while getopts cdi: flag; do
+    case $flag in
+        c)
+            boot="c$boot"
+            ;;
+        d)
+            boot="d$boot"
+            ;;
+        i)
+            iso="$OPTARG"
+            ;;
+        ?)
+            usage
+            ;;
+    esac
+done
+
+case $boot in
+    ?)
+        ;;
+    *)
+        usage
+        ;;
+esac
+
+set -e
+set -x
+
+case $boot in
+    c)
+        echo "booting CD with empty disk"
+        rm -f disk.img
+        dd if=/dev/zero "of=${DISK}" bs=1M count=0 seek=10240
+        cfg="-cdrom ${iso} -hda ${DISK} -boot d"
+        ;;
+    d)
+        echo "booting disk with no CD"
+        cfg="-hda ${DISK} -boot c"
+        ;;
+esac
+
+qemu-system-x86_64 -enable-kvm -m 1024 -vga vmware $cfg