Merge branch 'wattsworth'

README

@@ -1,33 +1,41 @@
-This live CD is customized by roughly following the guidelines at:
-  https://help.ubuntu.com/community/LiveCDCustomization
-  https://help.ubuntu.com/community/LiveCDCustomizationFromScratch
-  https://help.ubuntu.com/community/MakeALiveCD/DVD/BootableFlashFromHarddiskInstall
-plus a lot of other info, like ubiquity and casper source code, and
-additions for making the resulting iso EFI bootable.
+Tools to customize an Ubuntu Live CD.  Many parts of this will end up
+being version-specific.  The host machine must be running systemd,
+since the image is executed inside a systemd-nspawn container.
 
-extractiso.sh
-  extract the original iso
+Set up host machine, which must be running a recent systemd:
 
-customize.sh
-  apply customizations
+  sudo apt-get install squashfs-tools xorriso isolinux systemd-container iptables
+  sudo systemctl start systemd-networkd
 
-enter.sh
-  chroot into the install (for optional manual customization)
+Choose a config:
 
-buildiso.sh
-  build the new iso
+  export BUILD_CONFIG=16.04
 
-run.sh -c
-  boot the iso in qemu with a blank hdd, to test the install
+Download and extract the original ISO:
 
-run.sh -d
-  boot the installed hdd in qemu
+  ./extractiso.sh
 
-cleanup.sh
-  remove everything but the isos
+Apply customizations, both to the outer ISO and the inner image.
+This will also run customizer-inner.sh inside a container:
 
+  ./customize.sh
 
-SHORT VERSION:
-  sudo apt-get install squashfs-tools xorriso syslinux
-  time sudo sh -c "env BUILD_CONFIG=12.10 ./full-rebuild.sh"
-  time sudo sh -c "env BUILD_CONFIG=13.04 ./full-rebuild.sh"
+(Optional) Run and enter a shell in the container, for manual customization:
+
+  ./enter.sh
+
+Build the new ISO:
+
+  ./buildiso.sh
+
+Boot the ISO in QEMU with a blank HDD, to test and install:
+
+  ./run.sh -c
+
+Boot QEMU with the same HDD image as above, to test an "installed" system:
+
+  ./run.sh -d
+
+Remove everything but the ISOs:
+
+  ./cleanup.sh

buildiso.sh

@@ -1,56 +1,80 @@
 #!/bin/bash
 
-# build the iso
+# Build the iso.
+# Some parts of this (particularly the xorriso command lines) come from
+# the ubuntu-cdimage project:
+#   http://bazaar.launchpad.net/~ubuntu-cdimage/debian-cd/ubuntu/
+
+# make sure this was run as root
+if [ $UID -ne 0 ] ; then
+    echo "Need to be root; trying sudo"
+    exec sudo env BUILD_CONFIG=$BUILD_CONFIG $0 "$@"
+fi
 . config || exit 0
 
 set -x
 set -e
 
-if ! [ "$1" == "skip" ] ; then
+INITRD=${FS}/initrd.img
+VMLINUZ=$(readlink -f ${FS}/vmlinuz)
+VMLINUZ_EFI=${VMLINUZ}.efi.signed
+
+if ! [ "$1" == "justiso" ] ; then
 
-# copy kernel if changed
-if [ ${FS}/initrd.img -nt ${ISO}/casper/initrd.lz ] ; then
-    sudo cp ${FS}/vmlinuz ${ISO}/casper/vmlinuz
-    sudo sh -c "zcat ${FS}/initrd.img | lzma > ${ISO}/casper/initrd.lz"
-fi
 
-# manifests
-sudo chmod +w ${ISO}/casper/filesystem.manifest
-sudo chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
-    | sudo tee ${ISO}/casper/filesystem.manifest >/dev/null
+    if ! [ -r ${INITRD} -a -r ${VMLINUZ} -a -r ${VMLINUZ_EFI} ] ; then
+        set +x
+        echo "== Missing kernel and/or initrd.  Either upgrade or reinstall"
+        echo "== the kernel inside the image to get the right files."
+        exit 1
+    fi
 
-# squashfs
-sudo rm -f ${ISO}/casper/filesystem.squashfs
-sudo mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
-printf $(sudo du -sx --block-size=1 ${FS} | cut -f1) \
-    | sudo tee ${ISO}/casper/filesystem.size
+    # Recompress initrd if changed
+    if [ ${INITRD} -nt ${ISO}/casper/initrd.lz ] ; then
+        sh -c "zcat ${INITRD} | lzma > ${ISO}/casper/initrd.lz"
+    fi
+    # Get kernel and signed EFI kernel
+    cp ${VMLINUZ} ${ISO}/casper/vmlinuz
+    cp ${VMLINUZ_EFI} ${ISO}/casper/vmlinuz.efi
+
+    # manifests
+    chmod +w ${ISO}/casper/filesystem.manifest
+    chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
+        | tee ${ISO}/casper/filesystem.manifest >/dev/null
+
+    # squashfs
+    rm -f ${ISO}/casper/filesystem.squashfs
+    mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
+    printf $(du -sx --block-size=1 ${FS} | cut -f1) \
+        | tee ${ISO}/casper/filesystem.size
 
 fi
 
 # md5sums
-sudo rm -f md5sum.txt
-sudo find ${ISO} -type f -print0 \
-    | sudo xargs -0 md5sum \
+rm -f md5sum.txt
+find ${ISO} -type f -print0 \
+    | xargs -0 md5sum \
     | sed -e "s, ${ISO}, .," \
     | grep -v isolinux/boot.cat \
     | grep -v isolinux/isolinux.bin \
     | grep -v md5sum.txt \
-    | sudo tee ${ISO}/md5sum.txt >/dev/null
+    | tee ${ISO}/md5sum.txt >/dev/null
 
-sudo chown -R ${USER} ${ISO}
+chown -R ${NONPRIV_UID} ${ISO}
 
 # build CD
 xorriso -as mkisofs \
-    -D -r -V "NilmDBuntu ${VERSION}" -cache-inodes -J -l \
-    -input-charset utf-8 -o ${OUTPUT} \
+    -joliet -full-iso9660-filenames \
+    -rational-rock \
+    -V "NILMbuntu ${VERSION}" \
+    -input-charset utf-8 -output ${OUTPUT} \
     -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
     -boot-load-size 4 -boot-info-table \
     -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
+    -isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin \
+    -isohybrid-gpt-basdat -isohybrid-apm-hfsplus \
     ${ISO}
 
-# fix iso for hybrid booting
-isohybrid ${OUTPUT}
-
 set +x
 echo "Burn it with:"
 echo "  growisofs -dvd-compat -Z /dev/dvd=${OUTPUT}"

cleanup.sh

@@ -3,9 +3,4 @@
 # delete all temporary stuff
 . config || exit 0
 
-sudo umount ${FS}/dev/pts || true
-sudo umount ${FS}/sys/kernel/security || true
-sudo umount ${FS}/sys || true
-sudo umount ${FS}/proc || true
-sudo umount ${MNT} || true
 sudo rm --one-file-system -rf ${FS} ${ISO} ${MNT} ${DISK}

config

@@ -1,9 +1,13 @@
 #!/bin/bash
 
 if [ -z "$BUILD_CONFIG" ] ; then
-    echo "Set BUILD_CONFIG first"
-    exit 1
+    export BUILD_CONFIG=16.04
+    echo "No BUILD_CONFIG set -- assuming $BUILD_CONFIG"
 fi
 
 echo "Loading config file: config-$BUILD_CONFIG"
 . config-$BUILD_CONFIG
+#    echo " Ubuntu version: ${VERSION}"
+#    echo "      Input ISO: ${ISONAME}"
+#    echo "           from: ${ORIGURL}"
+#    echo "     Output ISO: ${OUTPUT}"

config-12.10

@@ -1,17 +0,0 @@
-# output image
-VERSION="12.10.2"
-OUTPUT="nilmdbuntu-${VERSION}.iso"
-
-# original ISOs
-ISONAME="xubuntu-12.10-desktop-amd64.iso"
-ORIG="tmp-12.10/$ISONAME"
-ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/12.10/release/$ISONAME"
-
-# work directories, temporary qemu HD image
-FS="tmp-12.10/fs"
-ISO="tmp-12.10/iso"
-MNT="tmp-12.10/mnt"
-DISK="tmp-12.10/disk.img"
-
-# local user
-USER=jim:jim

config-13.04

@@ -1,17 +0,0 @@
-# output image
-VERSION="13.04.2"
-OUTPUT="nilmdbuntu-${VERSION}.iso"
-
-# original ISOs
-ISONAME="xubuntu-13.04-desktop-amd64.iso"
-ORIG="tmp-13.04/$ISONAME"
-ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/13.04/release/$ISONAME"
-
-# work directories, temporary qemu HD image
-FS="tmp-13.04/fs"
-ISO="tmp-13.04/iso"
-MNT="tmp-13.04/mnt"
-DISK="tmp-13.04/disk.img"
-
-# local user
-USER=jim:jim

config-13.10

@@ -1,17 +0,0 @@
-# output image
-VERSION="13.10.2"
-OUTPUT="nilmdbuntu-${VERSION}.iso"
-
-# original ISOs
-ISONAME="xubuntu-13.10-desktop-amd64.iso"
-ORIG="tmp-13.10/$ISONAME"
-ORIGURL="http://hex1a4.net/xubuntu-13.10-desktop-amd64.iso"
-
-# work directories, temporary qemu HD image
-FS="tmp-13.10/fs"
-ISO="tmp-13.10/iso"
-MNT="tmp-13.10/mnt"
-DISK="tmp-13.10/disk.img"
-
-# local user
-USER=jim:jim

config-16.04

@@ -0,0 +1,19 @@
+# -*- sh -*-
+
+# output image
+VERSION="16.04"
+OUTPUT="nilmbuntu-${VERSION}.iso"
+
+# original ISO
+ISONAME="xubuntu-${VERSION}-desktop-amd64.iso"
+ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/${VERSION}/release/${ISONAME}"
+
+# work directories, temporary qemu HD image, etc.  Shouldn't need to change these.
+ORIG="tmp-${VERSION}/$ISONAME"
+FS="tmp-${VERSION}/fs"
+ISO="tmp-${VERSION}/iso"
+MNT="tmp-${VERSION}/mnt"
+DISK="tmp-${VERSION}/disk.img"
+
+# non-privileged user, so we don't need to do everything as root
+NONPRIV_UID=${SUDO_UID:-$UID}

customize-inner.sh

@@ -11,7 +11,9 @@ set -x
 try_install() {
     # try to install packages, but ignore failure
     for pkg in "$@"; do
-        apt-get -y install "$pkg" || true
+        if ! apt-get -y install "$pkg" ; then
+            echo ---- WARNING: Failed to install package: $pkg
+        fi
     done
 }
 
@@ -19,35 +21,23 @@ try_install() {
 cat >/etc/casper.conf <<"EOF"
 export USERNAME="ubuntu"
 export USERFULLNAME="Live session user"
-export HOST="nilmdb"
+export HOST="nilmbuntu"
 export BUILD_SYSTEM="Ubuntu"
-export FLAVOUR="NilmDBuntu"
+export FLAVOUR="NILMbuntu"
 EOF
 
-# In 13.10, this is needed to upgrade the "whoopsie" package; bug #1272269
-grep '13[.]10' /etc/issue && ln -sf /lib/init/upstart-job /etc/init.d/whoopsie
-
-# Upgrade packages, remove old kernels
+# Upgrade packages and remove old kernels
 apt-get update
-# in 13.04, doing upgrade & dist-upgrade together tries to install 2 kernels
-# at the same time, which breaks for some reason.  Also, try the upgrade
-# multiple times since that can help
-apt-get -y upgrade || apt-get -y upgrade || true
-apt-get -y dist-upgrade || apt-get -y dist-upgrade || true
+apt-get -y dist-upgrade
 apt-get -y --purge autoremove
 for VER in $(ls --sort=version /lib/modules/ | head -n -1) ; do
-    apt-get -y --purge remove ".*$VER.*"
+    apt-get -y --purge remove "linux-.*$VER"
 done
 
 # Disable upgrade popups
-sed -i -s -e 's/Prompt=normal/Prompt=never/g' \
+sed -i -s -e 's/Prompt=.*/Prompt=never/g' \
     /etc/update-manager/release-upgrades || true
 
-# some stuff we need from Ubuntu
-try_install \
-    wbritish \
-    thunderbird-locale-en-us
-
 # Set up & install postfix for local mail delivery
 debconf-set-selections <<"EOF"
 postfix postfix/mailname string localdomain
@@ -55,8 +45,9 @@ postfix postfix/main_mailer_type select Local only
 EOF
 apt-get -y install postfix
 
-# install nilmdb things
+# Required packages
 apt-get -y install \
+    python3 \
     python2.7 \
     python2.7-dev \
     python-setuptools \
@@ -80,301 +71,237 @@ apt-get -y install \
     python-scipy \
     python-daemon
 
-# install other useful but optional stuff
+# Install other useful but optional stuff
 try_install \
-    emacs-goodies-el \
-    emacs23-nox \
-    octave \
-    octave-signal \
-    octave-missing-functions \
-    gnuplot \
     curl \
+    devscripts \
+    dlocate \
+    emacs \
+    emacs-goodies-el \
+    gcc-arm-none-eabi \
+    gdb-arm-none-eabi \
     gddrescue \
+    gnuplot \
     help2man \
-    luatex \
-    pgf \
+    ipython \
+    libnewlib-arm-none-eabi \
+    libstdc++-arm-none-eabi-newlib \
+    texlive \
+    mailutils \
     moreutils \
-    ntfsprogs \
-    subversion \
-    dlocate \
-    ack-grep \
     mutt \
-    python-matplotlib \
-    ipython \
-    openvpn \
     network-manager-openvpn-gnome \
+    octave \
+    octave-missing-functions \
+    octave-signal \
+    openocd \
     openssl \
-    tcpdump \
+    openvpn \
+    python-matplotlib \
     screen \
-    devscripts \
-    mailutils
-
-# required
-apt-get -y install \
-    openssh-server
+    silversearcher-ag \
+    subversion \
+    tcpdump \
+    zip
 
 # Set up timezone to America/New_York for the live CD
-echo America/New_York > /etc/timezone
+ln -sf /usr/share/zoneinfo/America/New_York /etc/localtime
 dpkg-reconfigure -f noninteractive tzdata
 
-# Create nilmdb user to run the database
-adduser --system --group --shell /bin/bash --disabled-password nilmdb
-cp -rv /etc/skel/.??* /home/nilmdb
-chown -R nilmdb:nilmdb /home/nilmdb
-
-# Create WSGI scripts
-cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
-import nilmdb.server
-application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
-EOF
-cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
-import nilmrun.server
-application = nilmrun.server.wsgi_application("/nilmrun")
-EOF
-
-#### Edit apache config
-
-# Create apache config by hacking up the default one.  Might be a better way
-# to do this, and it'll probably break on new versions, but...
-
-APACHE_VER=$(dpkg -s apache2 | grep ^Version | cut -d ' ' -f 2)
-if dpkg --compare-versions $APACHE_VER ge 2.4 ; then
-    DEF=/etc/apache2/sites-available/000-default.conf
-    NEED_PERMISSIONS=1
-else
-    DEF=/etc/apache2/sites-available/default
-    NEED_PERMISSIONS=0
-fi
-
-# Cut out any existing NilmDB stuff
-perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig
-
-# Copy everything up to the first </VirtualHost> line
-perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF
-
-# Add the NilmDB config
-cat >>$DEF <<"EOF"
-	## NilmDB start
-	WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
-	WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
-	<Location /nilmdb>
-	    WSGIProcessGroup nilmdb-procgroup
-	    WSGIApplicationGroup nilmdb-appgroup
-	</Location>
-
-	WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
-	WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
-	<Location /nilmrun>
-	    WSGIProcessGroup nilmrun-procgroup
-	    WSGIApplicationGroup nilmrun-appgroup
-	</Location>
-EOF
-if [ $NEED_PERMISSIONS == 1 ] ; then
-cat >>$DEF <<"EOF"
-
-        <Directory /home/nilmdb>
-            Options All
-            AllowOverride All
-            Require all granted
-        </Directory>
-EOF
-fi
-cat >>$DEF <<"EOF"
-        ## NilmDB end
-EOF
-
-# Copy everything including and after the first </VirtualHost> line
-perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF
-
-#### Done editing apache config
-
-# Create nilmdb capture, processing, and cleanup files
-cat > /home/nilmdb/capture.sh <<"EOF"
-#!/bin/bash -e
-
-# Don't run capture if we're running off a live CD
-if grep -q boot=casper /proc/cmdline ; then
-    echo "Skipping capture, because this is a live CD."
-    exit 0
-fi
-
-echo "Starting capture in background..."
-nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
-    "ethstream -a 192.168.1.209 -n 6 -r 8000" \
-    "nilm-insert -m 10 -r 8000 --live /data/raw"
-EOF
-cat > /home/nilmdb/process.sh <<"EOF"
-#!/bin/bash -e
-
-# Ensure only one copy of this code runs at a time:
-LOCKFILE="/tmp/nilmdb-process.lock"
-exec 99>"$LOCKFILE"
-flock -n -x 99 || exit 0
-trap 'rm -f "$LOCKFILE"' 0
-
-nilm-sinefit -c 4 /data/raw /data/sinefit
-nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
-nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
-nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
-nilm-decimate-auto /data/raw "/data/prep*"
-nilm-cleanup --yes /home/nilmdb/cleanup.cfg
-EOF
-cat > /home/nilmdb/cleanup.cfg <<"EOF"
-[/data/prep-*]
-keep = 1y
-
-[/data/raw]
-keep = 2w
-
-[/data/sinefit]
-keep = 1y
-decimated = false
-EOF
-
-# Set up crontab
-cat > /home/nilmdb/crontab <<"EOF"
-SHELL=/bin/bash
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-# Run capture and processing scripts every 5 minutes
-*/5 * * * * chronic /home/nilmdb/capture.sh
-*/5 * * * * chronic /home/nilmdb/process.sh
-
-# Try to run nilmdb-fsck on boot.  It should hopefully run before
-# apache opens the database, and apache will return errors to clients
-# until nilmdb-fsck is done.
-@reboot chronic nilmdb-fsck --fix --no-data /home/nilmdb/db
-EOF
-crontab -u nilmdb /home/nilmdb/crontab
-
-# Fix permissions
-chown -R nilmdb:nilmdb /home/nilmdb
-chmod +x /home/nilmdb/{capture,process}.sh
-
-# Fetch and build everything.  Put it in the nilmdb dir
-echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
-GIT=https://git.jim.sh/jim/lees
-rm -rf /home/nilmdb/git
-mkdir /home/nilmdb/git
-chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
-REPOS="nilmdb nilmtools nilmrun ethstream"
-
-# check it out as nilmdb, so the .netrc gets used
-for repo in $REPOS; do
-    sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
-done
-
-# build as root, because we need to do that for the install
-for repo in $REPOS; do
-    make -C /home/nilmdb/git/$repo install
-done
-
-# fix up all permissions in git dir, so nilmdb user can play with it later
-chown -R nilmdb:nilmdb /home/nilmdb/git
-
-# Create the initial database and streams by running the standalone
-# server as nilmdb, making the right nilmtool calls, and killing it.
-sudo -i -u nilmdb nilmdb-server -a 127.0.0.1 -p 18646 -d /home/nilmdb/db &
-SERVERPID=$!
-trap "kill -9 $SERVERPID" 0
-for i in $(seq 1 120) ; do
-    sleep 1
-    echo waiting for nilmdb to start $i
-    if nilmtool -u http://127.0.0.1:18646/ info ; then
-        break
-    fi
-done
-nilmtool -u http://127.0.0.1:18646/ destroy -R "/data/*" || true
-nilmtool -u http://127.0.0.1:18646/ create /data/raw uint16_6
-nilmtool -u http://127.0.0.1:18646/ create /data/sinefit float32_3
-nilmtool -u http://127.0.0.1:18646/ create /data/prep-a float32_8
-nilmtool -u http://127.0.0.1:18646/ create /data/prep-b float32_8
-nilmtool -u http://127.0.0.1:18646/ create /data/prep-c float32_8
-kill $SERVERPID
-wait
-trap "" 0
-
 # Put some default desktop shortcuts in place
 DESKTOP=/etc/skel/Desktop
 mkdir -p $DESKTOP
 cp /usr/share/applications/exo-terminal-emulator.desktop $DESKTOP || true
 cp /usr/share/applications/exo-web-browser.desktop $DESKTOP || true
-chmod +x $DESKTOP/*  # needs to be executable for 13.04+
-
-# XFCE / theme customizations
-if [ -d /usr/share/themes/Clearlooks ] ; then
-    cat > /usr/share/gconf/defaults/88_nilmdbuntu-gtk-theme <<"EOF"
-/desktop/gnome/interface/gtk_theme "Clearlooks"
-EOF
-fi
-if [ -d /usr/share/icons/elementary-xfce ] ; then
-    cat > /usr/share/gconf/defaults/88_nilmdbuntu-icon-theme <<"EOF"
-/desktop/gnome/interface/icon_theme "elementary-xfce"
-EOF
-fi
-update-gconf-defaults
-
+chmod +x $DESKTOP/*
 
+# Custom background image (which was already copied in by customize.sh)
 XML=/etc/xdg/xdg-xubuntu/xfce4/xfconf/xfce-perchannel-xml
-BG=/usr/share/xfce4/backdrops
-mkdir -p $XML
-cat >$XML/xfce4-desktop.xml <<"EOF"
-<?xml version="1.0" encoding="UTF-8"?>
-
-<channel name="xfce4-desktop" version="1.0">
-  <property name="desktop-icons" type="empty">
-    <property name="style" type="int" value="2"/>
-    <property name="file-icons" type="empty">
-      <property name="show-home" type="bool" value="true"/>
-      <property name="show-filesystem" type="bool" value="true"/>
-      <property name="show-removable" type="bool" value="true"/>
-      <property name="show-trash" type="bool" value="true"/>
-    </property>
-  </property>
-  <property name="backdrop" type="empty">
-    <property name="screen0" type="empty">
-      <property name="monitor0" type="empty">
-        <property name="image-path" type="string"
-         value="/usr/share/xfce4/backdrops/nilmdbuntu.png"/>
-        <property name="image-show" type="bool" value="true"/>
-        <property name="image-style" type="int" value="4"/>
-        <property name="color-style" type="int" value="0"/>
-        <property name="color1" type="array">
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="65535"/>
-        </property>
-      </property>
-      <property name="monitor1" type="empty">
-        <property name="image-path" type="string"
-         value="/usr/share/xfce4/backdrops/nilmdbuntu.png"/>
-        <property name="image-show" type="bool" value="true"/>
-        <property name="image-style" type="int" value="4"/>
-        <property name="color-style" type="int" value="0"/>
-        <property name="color1" type="array">
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="65535"/>
-        </property>
-      </property>
-    </property>
-  </property>
-</channel>
-EOF
-sed -i -s -e 's/Greybird/Default/g' $XML/xfwm4.xml || true
-sed -i -s -e 's/Greybird/Clearlooks/g' $XML/xsettings.xml || true
-sed -i -s -e \
-    's/elementary-xfce-dark(er)?/elementary-xfce/g' $XML/xsettings.xml || true
-
-# Firefox defaults
-cat >/etc/firefox/syspref.js <<"EOF"
-pref("browser.startup.homepage", "http://nilmdb.com/");
-EOF
-cat >/etc/xul-ext/homepage.properties <<"EOF"
-browser.startup.homepage=http://nilmdb.com/
-EOF
-cat >/etc/xul-ext/ubufox.js <<"EOF"
-pref("browser.startup.homepage", "file:/etc/xul-ext/homepage.properties");
+sed -i -s -e 's/xubuntu-wallpaper.png/nilmbuntu.png/g' $XML/xfce4-desktop.xml
+
+# Configure Firefox with our default homepage
+HOMEPAGE="http://www.wattsworth.net/"
+cat >/etc/firefox/syspref.js <<EOF
+pref("browser.startup.homepage", "${HOMEPAGE}");
+pref("extensions.ubufox@ubuntu.com.custom_homepage", "${HOMEPAGE}");
+pref("browser.startup.homepage_override.mstone", "ignore");
 EOF
+
+# NilmDB setup below...
+
+# # Create nilmdb user to run the database
+# adduser --system --group --shell /bin/bash --disabled-password nilmdb
+# cp -rv /etc/skel/.??* /home/nilmdb
+# chown -R nilmdb:nilmdb /home/nilmdb
+
+# # Create WSGI scripts
+# cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
+# import nilmdb.server
+# application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
+# EOF
+# cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
+# import nilmrun.server
+# application = nilmrun.server.wsgi_application("/nilmrun")
+# EOF
+
+# #### Edit apache config
+
+# # Create apache config by hacking up the default one.  Might be a better way
+# # to do this, and it'll probably break on new versions, but...
+
+# APACHE_VER=$(dpkg -s apache2 | grep ^Version | cut -d ' ' -f 2)
+# if dpkg --compare-versions $APACHE_VER ge 2.4 ; then
+#     DEF=/etc/apache2/sites-available/000-default.conf
+#     NEED_PERMISSIONS=1
+# else
+#     DEF=/etc/apache2/sites-available/default
+#     NEED_PERMISSIONS=0
+# fi
+
+# # Cut out any existing NilmDB stuff
+# perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig
+
+# # Copy everything up to the first </VirtualHost> line
+# perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF
+
+# # Add the NilmDB config
+# cat >>$DEF <<"EOF"
+# 	## NilmDB start
+# 	WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
+# 	WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
+# 	<Location /nilmdb>
+# 	    WSGIProcessGroup nilmdb-procgroup
+# 	    WSGIApplicationGroup nilmdb-appgroup
+# 	</Location>
+
+# 	WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
+# 	WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
+# 	<Location /nilmrun>
+# 	    WSGIProcessGroup nilmrun-procgroup
+# 	    WSGIApplicationGroup nilmrun-appgroup
+# 	</Location>
+# EOF
+# if [ $NEED_PERMISSIONS == 1 ] ; then
+# cat >>$DEF <<"EOF"
+
+#         <Directory /home/nilmdb>
+#             Options All
+#             AllowOverride All
+#             Require all granted
+#         </Directory>
+# EOF
+# fi
+# cat >>$DEF <<"EOF"
+#         ## NilmDB end
+# EOF
+
+# # Copy everything including and after the first </VirtualHost> line
+# perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF
+
+# #### Done editing apache config
+
+# # Create nilmdb capture, processing, and cleanup files
+# cat > /home/nilmdb/capture.sh <<"EOF"
+# #!/bin/bash -e
+
+# # Don't run capture if we're running off a live CD
+# if grep -q boot=casper /proc/cmdline ; then
+#     echo "Skipping capture, because this is a live CD."
+#     exit 0
+# fi
+
+# echo "Starting capture in background..."
+# nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
+#     "ethstream -a 192.168.1.209 -n 6 -r 8000" \
+#     "nilm-insert -m 10 -r 8000 --live /data/raw"
+# EOF
+# cat > /home/nilmdb/process.sh <<"EOF"
+# #!/bin/bash -e
+
+# # Ensure only one copy of this code runs at a time:
+# LOCKFILE="/tmp/nilmdb-process.lock"
+# exec 99>"$LOCKFILE"
+# flock -n -x 99 || exit 0
+# trap 'rm -f "$LOCKFILE"' 0
+
+# nilm-sinefit -c 4 /data/raw /data/sinefit
+# nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
+# nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
+# nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
+# nilm-decimate-auto /data/raw "/data/prep*"
+# nilm-cleanup --yes /home/nilmdb/cleanup.cfg
+# EOF
+# cat > /home/nilmdb/cleanup.cfg <<"EOF"
+# [/data/prep-*]
+# keep = 1y
+
+# [/data/raw]
+# keep = 2w
+
+# [/data/sinefit]
+# keep = 1y
+# decimated = false
+# EOF
+
+# # Set up crontab
+# cat > /home/nilmdb/crontab <<"EOF"
+# SHELL=/bin/bash
+# PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# # Run capture and processing scripts every 5 minutes
+# */5 * * * * chronic /home/nilmdb/capture.sh
+# */5 * * * * chronic /home/nilmdb/process.sh
+
+# # Try to run nilmdb-fsck on boot.  It should hopefully run before
+# # apache opens the database, and apache will return errors to clients
+# # until nilmdb-fsck is done.
+# @reboot chronic nilmdb-fsck --fix --no-data /home/nilmdb/db
+# EOF
+# crontab -u nilmdb /home/nilmdb/crontab
+
+# # Fix permissions
+# chown -R nilmdb:nilmdb /home/nilmdb
+# chmod +x /home/nilmdb/{capture,process}.sh
+
+# # Fetch and build everything.  Put it in the nilmdb dir
+# echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
+# GIT=https://git.jim.sh/jim/lees
+# rm -rf /home/nilmdb/git
+# mkdir /home/nilmdb/git
+# chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
+# REPOS="nilmdb nilmtools nilmrun ethstream"
+
+# # check it out as nilmdb, so the .netrc gets used
+# for repo in $REPOS; do
+#     sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
+# done
+
+# # build as root, because we need to do that for the install
+# for repo in $REPOS; do
+#     make -C /home/nilmdb/git/$repo install
+# done
+
+# # fix up all permissions in git dir, so nilmdb user can play with it later
+# chown -R nilmdb:nilmdb /home/nilmdb/git
+
+# # Create the initial database and streams by running the standalone
+# # server as nilmdb, making the right nilmtool calls, and killing it.
+# sudo -i -u nilmdb nilmdb-server -a 127.0.0.1 -p 18646 -d /home/nilmdb/db &
+# SERVERPID=$!
+# trap "kill -9 $SERVERPID" 0
+# for i in $(seq 1 120) ; do
+#     sleep 1
+#     echo waiting for nilmdb to start $i
+#     if nilmtool -u http://127.0.0.1:18646/ info ; then
+#         break
+#     fi
+# done
+# nilmtool -u http://127.0.0.1:18646/ destroy -R "/data/*" || true
+# nilmtool -u http://127.0.0.1:18646/ create /data/raw uint16_6
+# nilmtool -u http://127.0.0.1:18646/ create /data/sinefit float32_3
+# nilmtool -u http://127.0.0.1:18646/ create /data/prep-a float32_8
+# nilmtool -u http://127.0.0.1:18646/ create /data/prep-b float32_8
+# nilmtool -u http://127.0.0.1:18646/ create /data/prep-c float32_8
+# kill $SERVERPID
+# wait
+# trap "" 0

customize.sh

@@ -6,16 +6,16 @@ set -x
 
 # Customize the outer ISO image
 sed -i -s -e \
-    "s/DISKNAME.*/DISKNAME  NilmDBuntu $VERSION by Jim Paris/" \
+    "s/DISKNAME.*/DISKNAME  NILMbuntu $VERSION/" \
     ${ISO}/README.diskdefines
 
 # The .disk/info file is important -- it's used by ubiquity to extract
 # out the distro name in dialogs, and I think casper might use it too
-echo "NilmDBuntu $VERSION by Jim Paris" > ${ISO}/.disk/info
+echo "NILMbuntu $VERSION" > ${ISO}/.disk/info
 
 # Set up preseed file
-cp ${ISO}/preseed/xubuntu.seed ${ISO}/preseed/nilmdbuntu.seed
-cat >> ${ISO}/preseed/nilmdbuntu.seed <<"EOF"
+cp ${ISO}/preseed/xubuntu.seed ${ISO}/preseed/nilmbuntu.seed
+cat >> ${ISO}/preseed/nilmbuntu.seed <<"EOF"
 # Commands to run on successful install:
 # - Regenerate SSH host keys
 # - Remove "quiet splash" from grub command line
@@ -35,7 +35,8 @@ ubiquity ubiquity/success_command string \
         umount /target/$i; done; \
     echo "success_command done" >>/target/var/log/installer/postinst.log
 
-# Default user.  Ubiquity should let them change this
+# Default user.  Ubiquity should let them change this.
+# (To skip the dialog completely, maybe set a password?)
 d-i passwd/user-fullname string NILM User
 d-i passwd/username string nilm
 #d-i passwd/user-password password nilm
@@ -45,24 +46,24 @@ d-i passwd/auto-login boolean true
 EOF
 
 # Set up isolinux how we want by editing its config
-TRY="Boot ^NilmDBuntu ${VERSION}"
+TRY="Boot ^NILMbuntu ${VERSION}"
 cp splash.png ${ISO}/isolinux/splash.png
 perl -n -i \
     -e '$n=1 if /^label/; $n=2 if /^label live-install/; next if $n==2;' \
-    -e "s/menu label.*(Try|NilmDBuntu).*/menu label $TRY/g;" \
-    -e "s,preseed/.*[.]seed,preseed/nilmdbuntu.seed,g;" \
+    -e "s/menu label.*(Try|NILM).*/menu label $TRY/g;" \
+    -e "s,preseed/.*[.]seed,preseed/nilmbuntu.seed,g;" \
     -e "s/ quiet splash//g;" \
     -e "print;" \
     ${ISO}/isolinux/txt.cfg
 sed -i -s -e "s/^ui gfxboot/# ui gfxboot/g;" ${ISO}/isolinux/isolinux.cfg
 
 # Set up grub similarly
-TRY="Boot NilmDBuntu ${VERSION}"
+TRY="Boot NILMbuntu ${VERSION}"
 perl -n -i \
     -e "next if /menuentry \"Install/../^}$/;" \
     -e "next if /menuentry \"OEM install/../^}$/;" \
-    -e "s/menuentry \"(Try|NilmDBuntu).*\" {/menuentry \"${TRY}\" {/g;" \
-    -e "s,preseed/.*[.]seed,preseed/nilmdbuntu.seed,g;" \
+    -e "s/menuentry \"(Try|NILM).*\" \{/menuentry \"${TRY}\" {/g;" \
+    -e "s,preseed/.*[.]seed,preseed/nilmbuntu.seed,g;" \
     -e "s/ quiet splash//g;" \
     -e "print;" \
     ${ISO}/boot/grub/{grub,loopback}.cfg
@@ -70,7 +71,7 @@ perl -n -i \
 if ! [ "$1" == "skip" ] ; then
 
 # Run the customize-inner.sh script inside the chroot
-sudo cp nilmdbuntu.png ${FS}/usr/share/xfce4/backdrops/nilmdbuntu.png
+sudo cp nilmbuntu.png ${FS}/usr/share/xfce4/backdrops/nilmbuntu.png
 sudo cp customize-inner.sh ${FS}/root/customize-inner.sh
 sudo chmod +x ${FS}/root/customize-inner.sh
 ./enter.sh "cd /root ; ./customize-inner.sh"

enter.sh

@@ -5,20 +5,91 @@ if [ $UID -ne 0 ] ; then
     echo "Need to be root; trying sudo"
     exec sudo env BUILD_CONFIG=$BUILD_CONFIG $0 "$@"
 fi
-
-# enter the chroot and run the command (if supplied) or a shell
 . config || exit 0
 
+# Spawn a systemd container that boots the machine, then run the given
+# command.  We can't execute systemd-nspawn directly, because that
+# only allows us to either boot the machine, or run a command (not
+# both).  Instead let's execute systemd-nspawn in a transient systemd
+# unit, then enter it using machinectl.
+
+UNIT=nilmbuntu-run-$VERSION
+MACH=nilmbuntu-$VERSION
+
+setup_networking() {
+    # We use a virtual ethernet adapter -- this requires that
+    # systemd-networkd is installed and running on the host.
+    if ! systemctl is-active systemd-networkd ; then
+        echo "Starting systemd-networkd"
+        systemctl start systemd-networkd
+    fi
+
+    # However, the current systemd-networkd from Debian is broken and
+    # won't enable masquerading -- so do it manually
+    IFACE=$(ip -4 route list default | head -1 | awk '{print $5}')
+    echo 1 > /proc/sys/net/ipv4/ip_forward
+    iptables -t nat -D POSTROUTING -o $IFACE -j MASQUERADE >/dev/null || true
+    iptables -t nat -A POSTROUTING -o $IFACE -j MASQUERADE
+}
+
+kill_container() {
+    # Kill any running container
+    if systemctl --quiet is-active $UNIT ; then
+        echo "Stopping container..."
+        # We could use "machinectl terminate", but that will wait
+        # for a clean shutdown or timeout; we don't need a clean
+        # shutdown, so send a SIGTERM twice to get systemd-nspawn
+        # to terminate pretty quickly
+        systemctl kill $UNIT
+        sleep 2
+        systemctl kill $UNIT
+
+        # Then wait for it to really stop
+        systemctl stop $UNIT
+    fi
+
+    # If systemd-nspawn returned with a failure code,
+    # the transient service unit file will stick around,
+    # so make sure we clear that.
+    if systemctl --quiet is-failed $UNIT ; then
+        systemctl reset-failed $UNIT
+    fi
+}
+
+start_container() {
+    kill_container
+    echo "Starting container..."
+    systemd-run --unit=$UNIT systemd-nspawn \
+                --quiet \
+                --keep-unit \
+                --boot \
+                --network-veth \
+                --directory $(realpath $FS) \
+                --machine $MACH
+    echo "Waiting..."
+    while ! env SYSTEMD_LOG_LEVEL=0 machinectl shell $MACH /bin/true ; do
+        sleep 0.1
+    done
+}
+
 FAILED=0
 run() {
+    # Run a command inside the container
     echo "+" "$1"
-    chroot ${FS} env -i \
-        HOME=/root \
-        PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
-        TERM=$TERM \
-        IN_CHROOT=1 \
-        bash -c "$1"
-    RET=$?
+
+    # machinectl doesn't propagate return codes, so append something
+    # to the command that saves the result of what we ran.
+    echo "99" > $FS/jim-cmd-result
+    CMD="$1 ; echo \$? > /jim-cmd-result"
+
+    # Run it
+    env SYSTEMD_LOG_LEVEL=notice machinectl \
+        shell $MACH /usr/bin/env IN_CHROOT=1 \
+        bash -c "$CMD"
+
+    # Check result
+    RET=$(cat $FS/jim-cmd-result)
+    rm -f $FS/jim-cmd-result
     if [ $RET -ne 0 ] && [ "$1" != "exec bash" ] ; then
         printf "%s\n" "----------- WARNING: failed with exit code $RET"
         FAILED=$RET
@@ -27,20 +98,19 @@ run() {
 }
 
 set -e
-umount ${FS}/dev/pts || true
-umount ${FS}/sys/kernel/security || true
-umount ${FS}/sys || true
-umount ${FS}/proc || true
-mount -t proc none ${FS}/proc
-mount -t sysfs none ${FS}/sys
-mount -t devpts none ${FS}/dev/pts
-
-run "echo 'nameserver 8.8.8.8' > /etc/resolv.conf"
-run "dbus-uuidgen > /var/lib/dbus/machine-id"
-run "dpkg-divert --local --rename --add /sbin/initctl"
-run "ln -sf /bin/true /sbin/initctl"
-run "dpkg-divert --local --rename --add /usr/sbin/update-grub"
-run "ln -sf /bin/true /usr/sbin/update-grub"
+
+setup_networking
+start_container
+
+run "resolvconf --disable-updates"
+run "echo 'nameserver 8.8.8.8' > /run/resolvconf/resolv.conf"
+run "hostnamectl --transient set-hostname nilmbuntu"
+
+#run "dbus-uuidgen > /var/lib/dbus/machine-id"
+#run "dpkg-divert --local --rename --add /sbin/initctl"
+#run "ln -sf /bin/true /sbin/initctl"
+#run "dpkg-divert --local --rename --add /usr/sbin/update-grub"
+#run "ln -sf /bin/true /usr/sbin/update-grub"
 
 set +e
 if [ -z "$1" ] ; then
@@ -48,21 +118,30 @@ if [ -z "$1" ] ; then
 else
     run "$1"
 fi
+
+echo "Cleaning up..."
+
+# Manually clean up some things that show up after booting an image
+# and installing packages.  This doesn't get everything, but what's
+# left should be fine.
 run "apt-get clean"
-run "rm /sbin/initctl"
-run "dpkg-divert --rename --remove /sbin/initctl"
-run "rm /usr/sbin/update-grub"
-run "dpkg-divert --rename --remove /usr/sbin/update-grub"
-run "rm /var/lib/dbus/machine-id"
-run "> /etc/resolv.conf"
-run "rm -rf /tmp/* /tmp/.??* /root/.bash_history"
-
-umount ${FS}/dev/pts
-umount ${FS}/sys/kernel/security || true
-umount ${FS}/sys
-umount ${FS}/proc
-
-echo "cleaned up"
+run "> /etc/machine-id"
+run "rm -f /core /boot/grub/grubenv"
+run "rm -f /var/lib/systemd/random-seed"
+run "rm -f /var/lib/ubuntu-drivers-common/last_gfx_boot"
+run "rm -f /var/lib/NetworkManager/*"
+run "rm -f /root/.bash_history"
+
+#run "rm /sbin/initctl"
+#run "dpkg-divert --rename --remove /sbin/initctl"
+#run "rm /usr/sbin/update-grub"
+#run "dpkg-divert --rename --remove /usr/sbin/update-grub"
+#run "rm /var/lib/dbus/machine-id"
+#run "> /etc/resolv.conf"
+
+kill_container
+
+echo "Done"
 if [ $FAILED -ne 0 ] ; then
     exit $FAILED
 fi

extractiso.sh

@@ -29,7 +29,7 @@ sudo mount -o loop,ro "$ORIG" ${MNT}
 # copy data
 sudo mkdir ${ISO}
 sudo rsync --exclude=/casper/filesystem.squashfs -a ${MNT}/ ${ISO}
-sudo chown -R ${USER} ${ISO}
+sudo chown -R ${NONPRIV_UID} ${ISO}
 chmod -R u+w ${ISO}
 
 # copy squashfs

full-rebuild.sh

@@ -11,13 +11,12 @@ fi
 
 set -e
 
-#rm -f ${OUTPUT}
 ./cleanup.sh
 for d in ${ORIG} ${OUTPUT} ${DISK} ; do
     mkdir -p $(dirname $d)
-    chown ${USER} $(dirname $d)
+    chown ${NONPRIV_UID} $(dirname $d)
 done
 ./extractiso.sh
 ./customize.sh
 ./buildiso.sh
-chown ${USER} ${ORIG} ${OUTPUT}
+chown ${NONPRIV_UID} ${ORIG} ${OUTPUT}

run.sh

@@ -45,17 +45,21 @@ esac
 set -e
 set -x
 
+cfg=""
 case $boot in
     c)
         echo "booting CD with empty disk"
         rm -f ${DISK}
         dd if=/dev/zero "of=${DISK}" bs=1M count=0 seek=10240
-        cfg="-cdrom ${iso} -hda ${DISK} -boot d"
+        cfg+=" -drive file=${iso},index=1,media=cdrom"
+        cfg+=" -drive file=${DISK},index=0,media=disk,format=raw"
+        cfg+=" -boot d"
         ;;
     d)
         echo "booting disk with no CD"
-        cfg="-hda ${DISK} -boot c"
+        cfg+=" -drive file=${DISK},index=0,media=disk,format=raw"
+        cfg+=" -boot c"
         ;;
 esac
 
-qemu-system-x86_64 -enable-kvm -m 1024 -vga vmware $cfg
+qemu-system-x86_64 -enable-kvm -m 2048 -vga vmware $cfg