Reduce VM disk size

If no upgrades were available, the existing package needs to be
reinstalled, because the images in /boot were erased as part of the
original live CD build process, and we rely on them being present to
fix up the ISO later.

.gitignore

@@ -1,9 +1,2 @@
-fs
+tmp-*
-fs-*
-iso
-iso-*
-mnt
-mnt-*
 *.iso
-disk.img
-disk.img-*

README

@@ -1,33 +0,0 @@
-This live CD is customized by roughly following the guidelines at:
-https://
-  help.ubuntu.com/community/LiveCDCustomization
-  help.ubuntu.com/community/LiveCDCustomizationFromScratch
-  help.ubuntu.com/community/MakeALiveCD/DVD/BootableFlashFromHarddiskInstall
-plus a lot of other info, like ubiquity and casper source code, and
-additions for making the resulting iso EFI bootable.
-
-extractiso.sh
-  extract the original iso
-
-customize.sh
-  apply customizations
-
-enter.sh
-  chroot into the install (for optional manual customization)
-
-buildiso.sh
-  build the new iso
-
-run.sh -c
-  boot the iso in qemu with a blank hdd, to test the install
-
-run.sh -d
-  boot the installed hdd in qemu
-
-cleanup.sh
-  remove everything but the isos
-
-
-SHORT VERSION:
-  time sudo sh -c "env BUILD_CONFIG=12.10 ./full-rebuild.sh"
-  time sudo sh -c "env BUILD_CONFIG=13.04 ./full-rebuild.sh"

README.md

@@ -0,0 +1,44 @@
+Nilmbuntu
+=========
+
+Tools to customize an Ubuntu Live CD.  Many parts of this will end up
+being version-specific.  The host machine must be running systemd,
+since the image is executed inside a systemd-nspawn container.
+
+Set up host machine:
+
+    sudo apt install squashfs-tools xorriso isolinux systemd-container iptables
+    sudo systemctl start systemd-networkd
+
+Choose a config:
+
+    export BUILD_CONFIG=20.04.1
+
+Download and extract the original ISO:
+
+    ./extractiso.sh
+
+Apply customizations, both to the outer ISO and the inner image.
+This will also run customizer-inner.sh inside a container:
+
+    ./customize.sh
+
+(Optional) Run and enter a shell in the container, for manual customization:
+
+    ./enter.sh
+
+Build the new ISO:
+
+    ./buildiso.sh
+
+Boot the ISO in QEMU with a blank HDD, to test and install:
+
+    ./run.sh -c
+
+Boot QEMU with the same HDD image as above, to test an "installed" system:
+
+    ./run.sh -d
+
+Remove everything but the ISOs:
+
+    ./cleanup.sh

TODO

@@ -1,15 +0,0 @@
-- Try other Ubuntu/Xubuntu versions
-
-Problems with 3ce08771a631f9e373f65995fd2114a817326e98:
-* Ubiquity chooser doesn't have nilmdb background --- skip it!
-* Theme is still "Greybird" not "Clearlooks"
-* Icons are still "elementary Xfce dark" not "elementary Xfce"
-* Change xfdesktop background color to black
-* Set default firefox homepage to http://nilmdb.com/
-* /home/nilmdb should have /etc/skel stuff
-* capture.sh should say skipping
-* ubiquity still shows release notes?
-* release notes need to exist
-* in live CD, ssh_host_dsa_key is ba636e7440..
-* quiet/splash removal failed  --- preseed late_command didn't work
-* build hybrid iso

backup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. config || exit 0
+set -e
+
+echo Backing up filesystem...
+sudo rsync -a --delete ${FS} ${FS}.backup

buildiso.sh

@@ -1,56 +1,74 @@
 #!/bin/bash
 
-# build the iso
+# Build the iso.
+# Some parts of this (particularly the xorriso command lines) come from
+# the ubuntu-cdimage project:
+#   http://bazaar.launchpad.net/~ubuntu-cdimage/debian-cd/ubuntu/
+
+# make sure this was run as root
+if [ $UID -ne 0 ] ; then
+    echo "Need to be root; trying sudo"
+    exec sudo env BUILD_CONFIG=$BUILD_CONFIG $0 "$@"
+fi
 . config || exit 0
 
 set -x
 set -e
 
-if ! [ "$1" == "skip" ] ; then
+INITRD=${FS}/boot/initrd.img
+VMLINUZ=${FS}/boot/vmlinuz
 
-# copy kernel if changed
+if ! [ "$1" == "justiso" ] ; then
-if [ ${FS}/initrd.img -nt ${ISO}/casper/initrd.lz ] ; then
-    sudo cp ${FS}/vmlinuz ${ISO}/casper/vmlinuz
-    sudo sh -c "zcat ${FS}/initrd.img | lzma > ${ISO}/casper/initrd.lz"
-fi
 
-# manifests
+    if ! [ -r ${INITRD} -a -r ${VMLINUZ} ] ; then
-sudo chmod +w ${ISO}/casper/filesystem.manifest
+        set +x
-sudo chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
+        echo "== Missing kernel and/or initrd.  Either upgrade or reinstall"
-    | sudo tee ${ISO}/casper/filesystem.manifest >/dev/null
+        echo "== the kernel inside the image to get the right files."
+        exit 1
+    fi
 
-# squashfs
+    # Copy kernel and initrd
-sudo rm -f ${ISO}/casper/filesystem.squashfs
+    cp ${INITRD} ${ISO}/casper/initrd
-sudo mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
+    cp ${VMLINUZ} ${ISO}/casper/vmlinuz
-printf $(sudo du -sx --block-size=1 ${FS} | cut -f1) \
+
-    | sudo tee ${ISO}/casper/filesystem.size
+    # manifests
+    chmod +w ${ISO}/casper/filesystem.manifest
+    chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
+        | tee ${ISO}/casper/filesystem.manifest >/dev/null
+
+    # squashfs
+    rm -f ${ISO}/casper/filesystem.squashfs
+    mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
+    printf $(du -sx --block-size=1 ${FS} | cut -f1) \
+        | tee ${ISO}/casper/filesystem.size
 
 fi
 
 # md5sums
-sudo rm -f md5sum.txt
+rm -f ${ISO}/md5sum.txt
-sudo find ${ISO} -type f -print0 \
+find ${ISO} -type f -print0 \
-    | sudo xargs -0 md5sum \
+    | xargs -0 md5sum \
     | sed -e "s, ${ISO}, .," \
     | grep -v isolinux/boot.cat \
     | grep -v isolinux/isolinux.bin \
     | grep -v md5sum.txt \
-    | sudo tee ${ISO}/md5sum.txt >/dev/null
+    | tee ${ISO}/md5sum.txt >/dev/null
 
-sudo chown -R ${USER} ${ISO}
+chown -R ${NONPRIV_UID} ${ISO}
 
 # build CD
 xorriso -as mkisofs \
-    -D -r -V "NilmDBuntu ${VERSION}" -cache-inodes -J -l \
+    -joliet -full-iso9660-filenames \
-    -input-charset utf-8 -o ${OUTPUT} \
+    -rational-rock \
+    -V "NILMbuntu ${VERSION}" \
+    -input-charset utf-8 -output ${OUTPUT} \
     -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
     -boot-load-size 4 -boot-info-table \
     -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
+    -isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin \
+    -isohybrid-gpt-basdat -isohybrid-apm-hfsplus \
     ${ISO}
 
-# fix iso for hybrid booting
-isohybrid ${OUTPUT}
-
 set +x
 echo "Burn it with:"
 echo "  growisofs -dvd-compat -Z /dev/dvd=${OUTPUT}"

cleanup.sh

@@ -3,9 +3,4 @@
 # delete all temporary stuff
 . config || exit 0
 
-sudo umount ${FS}/dev/pts || true
-sudo umount ${FS}/sys/kernel/security || true
-sudo umount ${FS}/sys || true
-sudo umount ${FS}/proc || true
-sudo umount mnt || true
 sudo rm --one-file-system -rf ${FS} ${ISO} ${MNT} ${DISK}

config

@@ -1,9 +1,13 @@
 #!/bin/bash
 
 if [ -z "$BUILD_CONFIG" ] ; then
-    echo "Set BUILD_CONFIG first"
+    export BUILD_CONFIG=20.04.1
-    exit 1
+    echo "No BUILD_CONFIG set -- assuming $BUILD_CONFIG"
 fi
 
 echo "Loading config file: config-$BUILD_CONFIG"
 . config-$BUILD_CONFIG
+#    echo " Ubuntu version: ${VERSION}"
+#    echo "      Input ISO: ${ISONAME}"
+#    echo "           from: ${ORIGURL}"
+#    echo "     Output ISO: ${OUTPUT}"

config-12.10

@@ -1,17 +0,0 @@
-# output image
-VERSION="12.10.1"
-OUTPUT="nilmdbuntu-${VERSION}.iso"
-
-# original ISOs
-ISONAME="xubuntu-12.10-desktop-amd64.iso"
-ORIG="tmp-12.10/$ISONAME"
-ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/12.10/release/$ISONAME"
-
-# work directories, temporary qemu HD image
-FS="tmp-12.10/fs"
-ISO="tmp-12.10/iso"
-MNT="tmp-12.10/mnt"
-DISK="tmp-12.10/disk.img"
-
-# local user
-USER=jim:jim

config-13.04

@@ -1,17 +0,0 @@
-# output image
-VERSION="13.04.1"
-OUTPUT="nilmdbuntu-${VERSION}.iso"
-
-# original ISOs
-ISONAME="xubuntu-13.04-desktop-amd64.iso"
-ORIG="tmp-13.04/$ISONAME"
-ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/13.04/release/$ISONAME"
-
-# work directories, temporary qemu HD image
-FS="tmp-13.04/fs"
-ISO="tmp-13.04/iso"
-MNT="tmp-13.04/mnt"
-DISK="tmp-13.04/disk.img"
-
-# local user
-USER=jim:jim

config-20.04.1

@@ -0,0 +1,19 @@
+# -*- sh -*-
+
+# output image
+VERSION="20.04.1"
+OUTPUT="nilmbuntu-${VERSION}.iso"
+
+# original ISO
+ISONAME="xubuntu-${VERSION}-desktop-amd64.iso"
+ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/${VERSION}/release/${ISONAME}"
+
+# work directories, temporary qemu HD image, etc.  Shouldn't need to change these.
+ORIG="tmp-${VERSION}/$ISONAME"
+FS="tmp-${VERSION}/fs"
+ISO="tmp-${VERSION}/iso"
+MNT="tmp-${VERSION}/mnt"
+DISK="tmp-${VERSION}/disk.img"
+
+# non-privileged user, so we don't need to do everything as root
+NONPRIV_UID=${SUDO_UID:-$UID}

customize-inner.sh

@@ -1,337 +0,0 @@
-#!/bin/bash
-
-if [ "$IN_CHROOT" != "1" ] ; then
-    echo This is supposed to run inside the chroot, oops
-    exit 1
-fi
-
-set -e
-set -x
-
-try_install() {
-    # try to install packages, but ignore failure
-    for pkg in "$@"; do
-        apt-get -y install "$pkg" || true
-    done
-}
-
-# Set up live username and hostname
-cat >/etc/casper.conf <<"EOF"
-export USERNAME="ubuntu"
-export USERFULLNAME="Live session user"
-export HOST="nilmdb"
-export BUILD_SYSTEM="Ubuntu"
-export FLAVOUR="NilmDBuntu"
-EOF
-
-# Upgrade packages, remove old kernels
-apt-get update
-# in 13.04, doing upgrade & dist-upgrade together tries to install 2 kernels
-# at the same time, which breaks for some reason.  Also, try the upgrade
-# multiple times since that can help
-apt-get -y upgrade || apt-get -y upgrade || true
-apt-get -y dist-upgrade || apt-get -y dist-upgrade || true
-apt-get -y --purge autoremove
-for VER in $(ls --sort=version /lib/modules/ | head -n -1) ; do
-    apt-get -y --purge remove ".*$VER.*"
-done
-
-# Disable upgrade popups
-sed -i -s -e 's/Prompt=normal/Prompt=never/g' \
-    /etc/update-manager/release-upgrades || true
-
-# some stuff we need from Ubuntu
-try_install \
-    wbritish \
-    thunderbird-locale-en-us
-
-# Set up & install postfix for local mail delivery
-debconf-set-selections <<"EOF"
-postfix postfix/mailname string localdomain
-postfix postfix/main_mailer_type select Local only
-EOF
-apt-get -y install postfix
-
-# install nilmdb things
-apt-get -y install \
-    python2.7 \
-    python2.7-dev \
-    python-setuptools \
-    python-pip \
-    cython \
-    git \
-    build-essential \
-    python-cherrypy3 \
-    python-decorator \
-    python-simplejson \
-    python-requests \
-    python-dateutil \
-    python-tz \
-    python-progressbar \
-    python-psutil \
-    python-numpy \
-    python-nose \
-    python-coverage \
-    apache2 \
-    libapache2-mod-wsgi \
-    python-scipy \
-    python-daemon
-
-# install other useful but optional stuff
-try_install \
-    emacs-goodies-el \
-    emacs23-nox \
-    octave \
-    octave-signal \
-    octave-missing-functions \
-    gnuplot \
-    curl \
-    gddrescue \
-    help2man \
-    luatex \
-    pgf \
-    moreutils \
-    ntfsprogs \
-    subversion \
-    dlocate \
-    ack-grep \
-    mutt \
-    python-matplotlib \
-    ipython \
-    openvpn \
-    network-manager-openvpn-gnome \
-    openssl
-
-# required
-apt-get -y install \
-    openssh-server
-
-# Set up timezone to America/New_York for the live CD
-echo America/New_York > /etc/timezone
-dpkg-reconfigure -f noninteractive tzdata
-
-# Create nilmdb user to run the database
-adduser --system --group --shell /bin/bash --disabled-password nilmdb
-cp -rv /etc/skel/.??* /home/nilmdb
-chown -R nilmdb:nilmdb /home/nilmdb
-
-# Create WSGI scripts
-cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
-import nilmdb.server
-application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
-EOF
-cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
-import nilmrun.server
-application = nilmrun.server.wsgi_application("/nilmrun")
-EOF
-
-# Create apache config by hacking up the default one.  Might be a better way
-# to do this, and it'll probably break on apache 2.4, but...
-DEF=/etc/apache2/sites-available/default
-perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig
-perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF
-cat >>$DEF <<"EOF"
-	## NilmDB start
-	WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
-	WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
-	<Location /nilmdb>
-	    WSGIProcessGroup nilmdb-procgroup
-	    WSGIApplicationGroup nilmdb-appgroup
-	</Location>
-
-	WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
-	WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
-	<Location /nilmrun>
-	    WSGIProcessGroup nilmrun-procgroup
-	    WSGIApplicationGroup nilmrun-appgroup
-	</Location>
-        ## NilmDB end
-EOF
-perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF
-
-# Create nilmdb capture, processing, and cleanup files
-cat > /home/nilmdb/capture.sh <<"EOF"
-#!/bin/bash -e
-
-# Don't run capture if we're running off a live CD
-if grep -q boot=casper /proc/cmdline ; then
-    echo "Skipping capture, because this is a live CD."
-    exit 0
-fi
-
-echo "Starting capture in background..."
-nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
-    "ethstream -a 192.168.1.209 -n 6 -r 8000" \
-    "nilm-insert -m 10 -r 8000 --live /data/raw"
-EOF
-cat > /home/nilmdb/process.sh <<"EOF"
-#!/bin/bash -e
-
-# Ensure only one copy of this code runs at a time:
-LOCKFILE="/tmp/nilmdb-process.lock"
-exec 99>"$LOCKFILE"
-flock -n -x 99 || exit 0
-trap 'rm -f "$LOCKFILE"' 0
-
-nilm-sinefit -c 4 /data/raw /data/sinefit
-nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
-nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
-nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
-nilm-decimate-auto /data/raw "/data/prep*"
-nilm-cleanup --yes /home/nilmdb/cleanup.cfg
-EOF
-cat > /home/nilmdb/cleanup.cfg <<"EOF"
-[/data/prep-*]
-keep = 1y
-
-[/data/raw]
-keep = 2w
-
-[/data/sinefit]
-keep = 1y
-decimated = false
-EOF
-
-# Set up crontab
-cat > /home/nilmdb/crontab <<"EOF"
-SHELL=/bin/bash
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
-# Run capture and processing scripts every 5 minutes
-*/5 * * * * chronic /home/nilmdb/capture.sh
-*/5 * * * * chronic /home/nilmdb/process.sh
-
-# Try to run nilmdb-fsck on boot.  It should hopefully run before
-# apache opens the database, and apache will return errors to clients
-# until nilmdb-fsck is done.
-@reboot chronic nilmdb-fsck --fix --no-data /home/nilmdb/db
-EOF
-crontab -u nilmdb /home/nilmdb/crontab
-
-# Fix permissions
-chown -R nilmdb:nilmdb /home/nilmdb
-chmod +x /home/nilmdb/{capture,process}.sh
-
-# Fetch and build everything.  Put it in the nilmdb dir
-echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
-GIT=https://git.jim.sh/jim/lees
-rm -rf /home/nilmdb/git
-mkdir /home/nilmdb/git
-chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
-REPOS="nilmdb nilmtools nilmrun ethstream"
-
-# check it out as nilmdb, so the .netrc gets used
-for repo in $REPOS; do
-    sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
-done
-
-# build as root, because we need to do that for the install
-for repo in $REPOS; do
-    make -C /home/nilmdb/git/$repo install
-done
-
-# fix up all permissions in git dir, so nilmdb user can play with it later
-chown -R nilmdb:nilmdb /home/nilmdb/git
-
-# Create the initial database and streams by running the standalone
-# server as nilmdb, making the right nilmtool calls, and killing it.
-sudo -i -u nilmdb nilmdb-server -a 127.0.0.1 -p 18646 -d /home/nilmdb/db &
-SERVERPID=$!
-trap "kill -9 $SERVERPID" 0
-for i in $(seq 1 120) ; do
-    sleep 1
-    echo waiting for nilmdb to start $i
-    if nilmtool -u http://127.0.0.1:18646/ info ; then
-        break
-    fi
-done
-nilmtool -u http://127.0.0.1:18646/ destroy -R "/data/*" || true
-nilmtool -u http://127.0.0.1:18646/ create /data/raw uint16_6
-nilmtool -u http://127.0.0.1:18646/ create /data/sinefit float32_3
-nilmtool -u http://127.0.0.1:18646/ create /data/prep-a float32_8
-nilmtool -u http://127.0.0.1:18646/ create /data/prep-b float32_8
-nilmtool -u http://127.0.0.1:18646/ create /data/prep-c float32_8
-kill $SERVERPID
-wait
-trap "" 0
-
-# Put some default desktop shortcuts in place
-DESKTOP=/etc/skel/Desktop
-mkdir -p $DESKTOP
-cp /usr/share/applications/exo-terminal-emulator.desktop $DESKTOP || true
-cp /usr/share/applications/exo-web-browser.desktop $DESKTOP || true
-chmod +x $DESKTOP/*  # needs to be executable for 13.04+
-
-# XFCE / theme customizations
-if [ -d /usr/share/themes/Clearlooks ] && \
-    [ -d /usr/share/icons/elementary-xfce ] ; then
-    cat > /usr/share/gconf/defaults/88_nilmdbuntu-settings <<"EOF"
-/desktop/gnome/interface/gtk_theme "Clearlooks"
-/desktop/gnome/interface/icon_theme "elementary-xfce"
-EOF
-    update-gconf-defaults
-fi
-
-XML=/etc/xdg/xdg-xubuntu/xfce4/xfconf/xfce-perchannel-xml
-BG=/usr/share/xfce4/backdrops
-mkdir -p $XML
-cat >$XML/xfce4-desktop.xml <<"EOF"
-<?xml version="1.0" encoding="UTF-8"?>
-
-<channel name="xfce4-desktop" version="1.0">
-  <property name="desktop-icons" type="empty">
-    <property name="style" type="int" value="2"/>
-    <property name="file-icons" type="empty">
-      <property name="show-home" type="bool" value="true"/>
-      <property name="show-filesystem" type="bool" value="true"/>
-      <property name="show-removable" type="bool" value="true"/>
-      <property name="show-trash" type="bool" value="true"/>
-    </property>
-  </property>
-  <property name="backdrop" type="empty">
-    <property name="screen0" type="empty">
-      <property name="monitor0" type="empty">
-        <property name="image-path" type="string"
-         value="/usr/share/xfce4/backdrops/nilmdbuntu.png"/>
-        <property name="image-show" type="bool" value="true"/>
-        <property name="image-style" type="int" value="4"/>
-        <property name="color-style" type="int" value="0"/>
-        <property name="color1" type="array">
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="65535"/>
-        </property>
-      </property>
-      <property name="monitor1" type="empty">
-        <property name="image-path" type="string"
-         value="/usr/share/xfce4/backdrops/nilmdbuntu.png"/>
-        <property name="image-show" type="bool" value="true"/>
-        <property name="image-style" type="int" value="4"/>
-        <property name="color-style" type="int" value="0"/>
-        <property name="color1" type="array">
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="0"/>
-          <value type="uint" value="65535"/>
-        </property>
-      </property>
-    </property>
-  </property>
-</channel>
-EOF
-sed -i -s -e 's/Greybird/Default/g' $XML/xfwm4.xml || true
-sed -i -s -e 's/Greybird/Clearlooks/g' $XML/xsettings.xml || true
-sed -i -s -e \
-    's/elementary-xfce-dark/elementary-xfce/g' $XML/xsettings.xml || true
-
-# Firefox defaults
-cat >/etc/firefox/syspref.js <<"EOF"
-pref("browser.startup.homepage", "http://nilmdb.com/");
-EOF
-cat >/etc/xul-ext/homepage.properties <<"EOF"
-browser.startup.homepage=http://nilmdb.com/
-EOF
-cat >/etc/xul-ext/ubufox.js <<"EOF"
-pref("browser.startup.homepage", "file:/etc/xul-ext/homepage.properties");
-EOF

customize.sh

@@ -6,73 +6,49 @@ set -x
 
 # Customize the outer ISO image
 sed -i -s -e \
-    "s/DISKNAME.*/DISKNAME  NilmDBuntu $VERSION by Jim Paris/" \
+    "s/DISKNAME.*/DISKNAME  NILMbuntu $VERSION/" \
     ${ISO}/README.diskdefines
 
 # The .disk/info file is important -- it's used by ubiquity to extract
 # out the distro name in dialogs, and I think casper might use it too
-echo "NilmDBuntu $VERSION by Jim Paris" > ${ISO}/.disk/info
+echo "NILMbuntu $VERSION" > ${ISO}/.disk/info
 
-# Set up preseed file
+# Set up preseed file by appending preseed.cfg to the Xubuntu
-cp ${ISO}/preseed/xubuntu.seed ${ISO}/preseed/nilmdbuntu.seed
+# default one
-cat >> ${ISO}/preseed/nilmdbuntu.seed <<"EOF"
+cat ${ISO}/preseed/xubuntu.seed files/preseed.cfg > ${ISO}/preseed/nilmbuntu.seed
-# Commands to run on successful install:
-# - Regenerate SSH host keys
-# - Remove "quiet splash" from grub command line
-# Ubiquity doesn't actually use preseed/late_command, so we do it
-# in ubiquity/success_command (which needs things mounted)
-ubiquity ubiquity/success_command string \
-    echo "success_command running" >/target/var/log/installer/postinst.log; \
-    for i in /dev /dev/pts /dev/shm /sys /sys/kernel/security /proc /cdrom; do \
-        mount --bind $i /target/$i; done; \
-    sed -i -e 's/quiet splash//g' /target/etc/default/grub; \
-    chroot /target update-grub \
-        >>/target/var/log/installer/postinst.log 2>&1; \
-    rm -f /target/etc/ssh/ssh_host_*; \
-    chroot /target dpkg-reconfigure openssh-server \
-        >>/target/var/log/installer/postinst.log 2>&1; \
-    for i in /dev/pts /dev/shm /dev /sys/kernel/security /sys /proc /cdrom; do \
-        umount /target/$i; done; \
-    echo "success_command done" >>/target/var/log/installer/postinst.log
-
-# Default user.  Ubiquity should let them change this
-d-i passwd/user-fullname string NILM User
-d-i passwd/username string nilm
-#d-i passwd/user-password password nilm
-#d-i passwd/user-password-again password nilm
-d-i user-setup/allow-password-weak boolean true
-d-i passwd/auto-login boolean true
-EOF
 
 # Set up isolinux how we want by editing its config
-TRY="Boot ^NilmDBuntu ${VERSION}"
+TRY="Boot ^NILMbuntu ${VERSION}"
-cp splash.png ${ISO}/isolinux/splash.png
+cp files/splash.png ${ISO}/isolinux/splash.png
 perl -n -i \
     -e '$n=1 if /^label/; $n=2 if /^label live-install/; next if $n==2;' \
-    -e "s/menu label.*(Try|NilmDBuntu).*/menu label $TRY/g;" \
+    -e "s/menu label.*Try Xubuntu without installing/menu label $TRY/g;" \
-    -e "s,preseed/.*[.]seed,preseed/nilmdbuntu.seed,g;" \
+    -e "s,preseed/.*[.]seed,preseed/nilmbuntu.seed,g;" \
     -e "s/ quiet splash//g;" \
     -e "print;" \
     ${ISO}/isolinux/txt.cfg
+sed -i -s -e "s/^menu margin .*/menu margin 2/g;" ${ISO}/isolinux/menu.cfg
 sed -i -s -e "s/^ui gfxboot/# ui gfxboot/g;" ${ISO}/isolinux/isolinux.cfg
 
 # Set up grub similarly
-TRY="Boot NilmDBuntu ${VERSION}"
+TRY="Boot NILMbuntu ${VERSION}"
 perl -n -i \
     -e "next if /menuentry \"Install/../^}$/;" \
     -e "next if /menuentry \"OEM install/../^}$/;" \
-    -e "s/menuentry \"(Try|NilmDBuntu).*\" {/menuentry \"${TRY}\" {/g;" \
+    -e "s/Try Xubuntu without installing/${TRY}/g;" \
-    -e "s,preseed/.*[.]seed,preseed/nilmdbuntu.seed,g;" \
+    -e "s,preseed/.*[.]seed,preseed/nilmbuntu.seed,g;" \
     -e "s/ quiet splash//g;" \
     -e "print;" \
     ${ISO}/boot/grub/{grub,loopback}.cfg
 
+# Prevent some packages from being removed after install
+perl -n -i -e 'print unless /gparted/' ${ISO}/casper/filesystem.manifest-remove
+
+sudo install -D -m 0644 files/fallback_dns.conf ${FS}/etc/systemd/resolved.conf.d/fallback_dns.conf
+
 if ! [ "$1" == "skip" ] ; then
 
 # Run the customize-inner.sh script inside the chroot
-sudo cp nilmdbuntu.png ${FS}/usr/share/xfce4/backdrops/nilmdbuntu.png
+./enter.sh "cd /root ; files/customize-inner.sh"
-sudo cp customize-inner.sh ${FS}/root/customize-inner.sh
-sudo chmod +x ${FS}/root/customize-inner.sh
-./enter.sh "cd /root ; ./customize-inner.sh"
 
 fi

enter.sh

@@ -5,20 +5,91 @@ if [ $UID -ne 0 ] ; then
     echo "Need to be root; trying sudo"
     exec sudo env BUILD_CONFIG=$BUILD_CONFIG $0 "$@"
 fi
-
-# enter the chroot and run the command (if supplied) or a shell
 . config || exit 0
 
+# Spawn a systemd container that boots the machine, then run the given
+# command.  We can't execute systemd-nspawn directly, because that
+# only allows us to either boot the machine, or run a command (not
+# both).  Instead let's execute systemd-nspawn in a transient systemd
+# unit, then enter it using machinectl.
+
+UNIT=nilmbuntu-run-$NONPRIV_UID-$VERSION
+MACH=nilmbuntu-$NONPRIV_UID-$VERSION
+
+setup_networking() {
+    # We use a virtual ethernet adapter -- this requires that
+    # systemd-networkd is installed and running on the host.
+    if ! systemctl is-active systemd-networkd ; then
+        echo "Starting systemd-networkd"
+        systemctl start systemd-networkd
+    fi
+
+    # However, the current systemd-networkd from Debian is broken and
+    # won't enable masquerading -- so do it manually
+#    IFACE=$(ip -4 route list default | head -1 | awk '{print $5}')
+#    echo 1 > /proc/sys/net/ipv4/ip_forward
+#    iptables -t nat -D POSTROUTING -o $IFACE -j MASQUERADE >/dev/null || true
+#    iptables -t nat -A POSTROUTING -o $IFACE -j MASQUERADE
+}
+
+kill_container() {
+    # Kill any running container
+    if systemctl --quiet is-active $UNIT ; then
+        echo "Stopping container..."
+        # We could use "machinectl terminate", but that will wait
+        # for a clean shutdown or timeout; we don't need a clean
+        # shutdown, so send a SIGTERM twice to get systemd-nspawn
+        # to terminate pretty quickly
+        systemctl kill $UNIT
+        sleep 2
+        systemctl kill $UNIT
+
+        # Then wait for it to really stop
+        systemctl stop $UNIT
+    fi
+
+    # If systemd-nspawn returned with a failure code,
+    # the transient service unit file will stick around,
+    # so make sure we clear that.
+    if systemctl --quiet is-failed $UNIT ; then
+        systemctl reset-failed $UNIT
+    fi
+}
+
+start_container() {
+    kill_container
+    echo "Starting container..."
+    systemd-run --unit=$UNIT systemd-nspawn \
+                --quiet \
+                --keep-unit \
+                --boot \
+                --network-veth \
+                --directory $(realpath $FS) \
+                --machine $MACH
+    echo "Waiting..."
+    while ! env SYSTEMD_LOG_LEVEL=0 machinectl shell $MACH /bin/true ; do
+        sleep 0.1
+    done
+}
+
 FAILED=0
 run() {
+    # Run a command inside the container
     echo "+" "$1"
-    chroot ${FS} env -i \
+
-        HOME=/root \
+    # machinectl doesn't propagate return codes, so append something
-        PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+    # to the command that saves the result of what we ran.
-        TERM=$TERM \
+    echo "99" > $FS/jim-cmd-result
-        IN_CHROOT=1 \
+    CMD="$1 ; echo \$? > /jim-cmd-result"
-        bash -c "$1"
+
-    RET=$?
+    # Run it
+    env SYSTEMD_LOG_LEVEL=notice machinectl \
+        shell $MACH /usr/bin/env IN_CHROOT=1 \
+        bash -c "$CMD"
+
+    # Check result
+    RET=$(cat $FS/jim-cmd-result)
+    rm -f $FS/jim-cmd-result
     if [ $RET -ne 0 ] && [ "$1" != "exec bash" ] ; then
         printf "%s\n" "----------- WARNING: failed with exit code $RET"
         FAILED=$RET
@@ -27,16 +98,14 @@ run() {
 }
 
 set -e
-mount -t proc none ${FS}/proc
-mount -t sysfs none ${FS}/sys
-mount -t devpts none ${FS}/dev/pts
 
-run "echo 'nameserver 8.8.8.8' > /etc/resolv.conf"
+rsync -avP --delete files/ ${FS}/root/files/
-run "dbus-uuidgen > /var/lib/dbus/machine-id"
+
-run "dpkg-divert --local --rename --add /sbin/initctl"
+setup_networking
-run "ln -s /bin/true /sbin/initctl"
+start_container
-run "dpkg-divert --local --rename --add /usr/sbin/update-grub"
+
-run "ln -s /bin/true /usr/sbin/update-grub"
+run "cat /etc/hosts.nilm >>/etc/hosts 2>/dev/null || true"
+run "hostnamectl --transient set-hostname nilmbuntu"
 
 set +e
 if [ -z "$1" ] ; then
@@ -44,21 +113,25 @@ if [ -z "$1" ] ; then
 else
     run "$1"
 fi
-run "apt-get clean"
-run "rm /sbin/initctl"
-run "dpkg-divert --rename --remove /sbin/initctl"
-run "rm /usr/sbin/update-grub"
-run "dpkg-divert --rename --remove /usr/sbin/update-grub"
-run "rm /var/lib/dbus/machine-id"
-run "> /etc/resolv.conf"
-run "rm -rf /tmp/* /tmp/.??* /root/.bash_history"
 
-umount ${FS}/dev/pts
+echo "Cleaning up..."
-umount ${FS}/sys/kernel/security || true
-umount ${FS}/sys
-umount ${FS}/proc
 
-echo "cleaned up"
+# Manually clean up some things that show up after booting an image
+# and installing packages.  This doesn't get everything, but what's
+# left should be fine.
+run "apt clean"
+run "dpkg --clear-avail"
+run "> /etc/machine-id"
+run "rm -f /core /boot/grub/grubenv"
+run "rm -f /var/lib/systemd/random-seed"
+run "rm -f /var/lib/ubuntu-drivers-common/last_gfx_boot"
+run "rm -f /var/lib/NetworkManager/*"
+run "rm -f /root/.bash_history"
+run "rm -rf /root/files"
+
+kill_container
+
+echo "Done"
 if [ $FAILED -ne 0 ] ; then
     exit $FAILED
 fi

extractiso.sh

@@ -17,7 +17,13 @@ set -e
 # download it if it doesn't exist
 if ! [ -e ${ORIG} ] ; then
     mkdir -p $(dirname ${ORIG})
-    wget -O "${ORIG}" "${ORIGURL}"
+    if [ -e $(dirname ${ORIG})/../$(basename ${ORIG}) ] ; then
+        # grab from parent directory
+        cp $(dirname ${ORIG})/../$(basename ${ORIG}) ${ORIG}
+    else
+        # grab from web
+        wget -O "${ORIG}" "${ORIGURL}"
+    fi
 fi
 
 # mount it
@@ -29,7 +35,7 @@ sudo mount -o loop,ro "$ORIG" ${MNT}
 # copy data
 sudo mkdir ${ISO}
 sudo rsync --exclude=/casper/filesystem.squashfs -a ${MNT}/ ${ISO}
-sudo chown -R ${USER} ${ISO}
+sudo chown -R ${NONPRIV_UID} ${ISO}
 chmod -R u+w ${ISO}
 
 # copy squashfs

files/casper.conf

@@ -0,0 +1,18 @@
+export USERNAME="nilm"
+export USERFULLNAME="NILM User"
+export HOST="nilmbuntu-live"
+export BUILD_SYSTEM="Ubuntu"
+export FLAVOUR="NILMbuntu"
+
+# Do the inital user setup even though UID 1000 already exists.
+# This also needs to be set in /etc/environment, for ubiquity,
+# but we do that in preseed/early_command so that it only shows
+# up in the LiveCD image.
+export OVERRIDE_SYSTEM_USER=true
+
+# The ISO has a UUID stored in /.disk/casper-uuid-generic, which
+# won't match the new UUID that update-initramfs put in /conf/uuid.conf,
+# so it won't find the live media at casper boot.
+# Easiest way to avoid this is to set a blank UUID, so that the
+# UUID checks are skipped.
+export UUID=""

files/customize-inner.sh

@@ -0,0 +1,140 @@
+#!/bin/bash
+
+if [ "$IN_CHROOT" != "1" ] ; then
+    echo This is supposed to run inside the chroot, oops
+    exit 1
+fi
+
+set -e
+set -x
+
+# Move stuff to the right places
+install -D -m 0644 files/nilmbuntu.png /usr/share/xfce4/backdrops/nilmbuntu.png
+install -D -m 0644 files/casper.conf /etc/casper.conf
+install -D -m 0644 files/hosts.nilm /etc/hosts.nilm
+
+# Disable apport crash reporting
+sed -i -s -e 's/enabled=1/enabled=0/g' /etc/default/apport
+
+# Copy hosts for now (although it will get overwritten at boot)
+cat /etc/hosts.nilm >> /etc/hosts
+
+# Upgrade packages
+apt update
+apt -y dist-upgrade
+apt -y --purge autoremove
+
+# Remove old kernels, and reinstall the latest one (to make sure
+# it's present in /boot)
+LATEST_KERNEL=$(ls --sort=version /lib/modules/ | tail -n 1)
+for VER in $(ls --sort=version /lib/modules/) ; do
+    if [ $VER != $LATEST_KERNEL ] ; then
+        apt -y --purge remove "linux-.*$VER"
+    fi
+done
+apt -y --reinstall install linux-image-${LATEST_KERNEL}
+
+# Disable upgrade popups
+sed -i -s -e 's/Prompt=.*/Prompt=never/g' \
+    /etc/update-manager/release-upgrades || true
+
+# Set up & install postfix for local mail delivery
+debconf-set-selections <<"EOF"
+postfix postfix/mailname string localdomain
+postfix postfix/main_mailer_type select Local only
+EOF
+apt -y install postfix
+
+# Required packages
+apt -y install \
+    python3 \
+    python3-venv \
+    python3-pip \
+    git \
+    build-essential \
+    apache2 \
+    libapache2-mod-wsgi
+
+# Install other useful stuff
+apt -y install \
+    adb \
+    avrdude \
+    curl \
+    debconf-utils \
+    devscripts \
+    dfu-programmer \
+    dlocate \
+    emacs \
+    emacs-goodies-el \
+    esptool \
+    firefox \
+    flashrom \
+    gddrescue \
+    git \
+    gnuplot \
+    gparted \
+    help2man \
+    ipython3 \
+    libnewlib-arm-none-eabi \
+    libreoffice \
+    libstdc++-arm-none-eabi-newlib \
+    texlive \
+    mailutils \
+    moreutils \
+    mutt \
+    octave \
+    octave-missing-functions \
+    octave-signal \
+    openocd \
+    openssh-server \
+    openssl \
+    openvpn \
+    screen \
+    silversearcher-ag \
+    tcpdump \
+    zip
+
+# Install the packages required for en_US language support
+# This avoids a prompt and package download at first install
+apt -y install $(/usr/bin/check-language-support -l en_US)
+
+# Set up timezone to America/New_York for the live CD
+ln -sf /usr/share/zoneinfo/America/New_York /etc/localtime
+dpkg-reconfigure -f noninteractive tzdata
+
+# Put some default desktop shortcuts in place
+DESKTOP=/etc/skel/Desktop
+mkdir -p $DESKTOP
+cp /usr/share/applications/exo-terminal-emulator.desktop $DESKTOP || true
+cp /usr/share/applications/exo-web-browser.desktop $DESKTOP || true
+chmod +x $DESKTOP/*
+
+# Custom background image
+XML=/etc/xdg/xdg-xubuntu/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml
+BG=/usr/share/xfce4/backdrops/nilmbuntu.png
+apt -y install xmlstarlet
+xmlstarlet ed -L -u "//property[@name='image-path']/@value" -v "$BG" $XML
+xmlstarlet ed -L -u "//property[@name='image-style']/@value" -v "3" $XML
+
+# Make Firefox nicer
+install -D -m 0644 files/syspref.js /etc/firefox/syspref.js
+
+# Create NILM user.  This should happen after anything that we put in
+# /etc/skel, since files get copied from there.  Note that this user
+# is the same as what we have in the preseed file and casper.conf
+if ! getent passwd nilm ; then
+   adduser --gecos "NILM User" --disabled-password nilm
+fi
+
+# Make sure locate databases are up-to-date
+/etc/cron.daily/mlocate
+/etc/cron.daily/dlocate
+
+# Make sure initramfs was regenerated with casper changes.
+# Manipulate some config files to avoid host stuff leaking through.
+echo 'RESUME=none' > /etc/initramfs-tools/conf.d/resume
+mv /etc/crypttab /etc/crypttab-old
+CASPER_GENERATE_UUID=1 update-initramfs -u
+rm /etc/initramfs-tools/conf.d/resume
+mv /etc/crypttab-old /etc/crypttab
+

files/fallback_dns.conf

@@ -0,0 +1,2 @@
+[Resolve]
+FallbackDNS=1.1.1.1 9.9.9.10 8.8.8.8 2606:4700:4700::1111 2620:fe::10 2001:4860:4860::8888

files/hosts.nilm

@@ -0,0 +1,2 @@
+127.0.2.1 nilm.primary
+127.0.2.2 nilm.secondary

files/preseed.cfg

@@ -0,0 +1,58 @@
+# Commands to run at casper boot:
+# - Regenerate SSH keys
+# - Append /etc/hosts.nilm to /etc/hosts
+#
+# And the following hacks, which could also run in customize-inner.sh,
+# but doing them here ensures that they'll only be present during the
+# LiveCD session
+# - Append some stuff to /etc/enviroment that's needed for Ubiquity to
+#   create the initial user properly, when we've already created it.
+# - Hack Ubiquity to prevent changing the username, and to set
+#   a default password
+d-i preseed/early_command string \
+    echo "early_command running" ; \
+    chroot /root sh -c "rm -f /etc/ssh/ssh_host_*" ; \
+    chroot /root sh -c "dpkg-reconfigure openssh-server" ; \
+    cat /root/etc/hosts.nilm >> /root/etc/hosts ; \
+    echo OVERRIDE_SYSTEM_USER=1 >> /root/etc/environment ; \
+    sed -i \
+        -e 's@id="login_encrypt">@id="login_encrypt"><property name="sensitive">False</property>@g' \
+        -e 's@id="fullname">@id="fullname"><property name="sensitive">False</property>@g' \
+        -e 's@id="username">@id="username"><property name="sensitive">False</property>@g' \
+        -e 's@"visibility">False@"visibility">True@g' \
+        -e 's@id="password">@id="password"><property name="text">nilm</property>@g' \
+        -e 's@id="verified_password">@id="verified_password"><property name="text">nilm</property>@g' \
+        /root/usr/share/ubiquity/gtk/stepUserInfo.ui ; \
+    echo "early_command done"
+#    chroot /root sh -c "echo UBIQUITY_AUTOMATIC=1 >> /etc/environment"
+
+# Commands to run on successful install:
+#   (Ubiquity doesn't actually use preseed/late_command, so we do it
+#    in ubiquity/success_command (which needs things mounted))
+# - Remove "quiet splash" from grub command line
+# - Append /etc/hosts.nilm to /etc/hosts
+ubiquity ubiquity/success_command string \
+    echo "success_command running" >/target/var/log/installer/postinst.log; \
+    mount --bind /dev /target/dev ; \
+    in-target sh -c "sed -i -e 's/quiet splash//g' /etc/default/grub" ; \
+    in-target sh -c "update-grub >>/var/log/installer/postinst.log 2>&1" ; \
+    in-target sh -c "cat /etc/hosts.nilm >> /etc/hosts " ; \
+    umount /target/dev ; \
+    echo "success_command done" >>/target/var/log/installer/postinst.log
+
+ubiquity ubiquity/use_nonfree boolean true
+
+# Default user.  Ubiquity should let them change this.
+# (To skip the dialog completely, set the first 4, and put
+# UBIQUITY_AUTOMATIC in /etc/environment in early_command above)
+d-i passwd/user-fullname string NILM User
+d-i passwd/username string nilm
+#d-i passwd/user-password password nilm
+#d-i passwd/user-password-again password nilm
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/force-encrypt-home boolean false
+d-i user-setup/encrypt-home boolean false
+d-i passwd/auto-login boolean true
+
+d-i netcfg/get_hostname seen true
+d-i netcfg/get_hostname string nilmbuntu

files/syspref.js

@@ -0,0 +1,16 @@
+pref("browser.aboutwelcome.enabled", false);
+pref("datareporting.policy.firstRunURL", "");
+pref("network.trr.mode", 5);
+pref("toolkit.telemetry.reportingpolicy.firstRun", false);
+pref("browser.startup.homepage_override.mstone", "ignore");
+pref("trailhead.firstrun.didSeeAboutWelcome", true);
+
+pref("browser.newtabpage.activity-stream.default.sites","https://www.google.com/");
+pref("browser.newtabpage.activity-stream.feeds.asrouterfeed", false);
+pref("browser.newtabpage.activity-stream.feeds.discoverystreamfeed", false);
+pref("browser.newtabpage.activity-stream.feeds.section.highlights", false);
+pref("browser.newtabpage.activity-stream.feeds.section.topstories", false);
+pref("browser.newtabpage.activity-stream.feeds.snippets", false);
+pref("browser.newtabpage.activity-stream.section.highlights.includePocket", false);
+pref("browser.newtabpage.pinned", "[{\"url\":\"https://www.wattsworth.net\",\"label\":\"Wattsworth\"},{\"url\":\"http://nilm.primary\",\"label\":\"Local NILM\"}]");
+pref("browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts", false);

full-rebuild.sh

@@ -11,13 +11,12 @@ fi
 
 set -e
 
-#rm -f ${OUTPUT}
 ./cleanup.sh
 for d in ${ORIG} ${OUTPUT} ${DISK} ; do
     mkdir -p $(dirname $d)
-    chown ${USER} $(dirname $d)
+    chown ${NONPRIV_UID} $(dirname $d)
 done
 ./extractiso.sh
 ./customize.sh
 ./buildiso.sh
-chown ${USER} ${ORIG} ${OUTPUT}
+chown ${NONPRIV_UID} ${ORIG} ${OUTPUT}

restore.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. config || exit 0
+set -e
+
+echo Restoring filesystem...
+sudo rsync -a --delete ${FS}.backup ${FS}

run.sh

@@ -45,17 +45,32 @@ esac
 set -e
 set -x
 
+cfg=""
+
+cfg+=" -nodefaults"
+cfg+=" -drive file=${DISK},media=disk,format=raw,if=virtio"
+cfg+=" -enable-kvm"
+cfg+=" -m 2048"
+cfg+=" -usb"
+cfg+=" -device usb-tablet"
+cfg+=" -nic user"
+cfg+=" -vga virtio"
+cfg+=" -k en-us"
+#cfg+=" -vnc :0"
+
 case $boot in
     c)
         echo "booting CD with empty disk"
         rm -f ${DISK}
-        dd if=/dev/zero "of=${DISK}" bs=1M count=0 seek=10240
+        dd if=/dev/zero "of=${DISK}" bs=1M count=0 seek=24576
-        cfg="-cdrom ${iso} -hda ${DISK} -boot d"
+        cfg+=" -drive file=${iso},media=cdrom,if=none,id=cd"
+        cfg+=" -device virtio-scsi-pci -device scsi-cd,drive=cd"
+        cfg+=" -boot d"
         ;;
     d)
         echo "booting disk with no CD"
-        cfg="-hda ${DISK} -boot c"
+        cfg+=" -boot c"
         ;;
 esac
 
-qemu-system-x86_64 -enable-kvm -m 1024 -vga vmware $cfg
+qemu-system-x86_64 $cfg