Browse Source

Merge branch 'wattsworth'

tags/nilmbuntu-5
Jim Paris 5 years ago
parent
commit
b693071019
16 changed files with 470 additions and 461 deletions
  1. +32
    -24
      README
  2. +50
    -26
      buildiso.sh
  3. +0
    -5
      cleanup.sh
  4. +6
    -2
      config
  5. +0
    -17
      config-12.10
  6. +0
    -17
      config-13.04
  7. +0
    -17
      config-13.10
  8. +19
    -0
      config-16.04
  9. +224
    -297
      customize-inner.sh
  10. +13
    -12
      customize.sh
  11. +116
    -37
      enter.sh
  12. +1
    -1
      extractiso.sh
  13. +2
    -3
      full-rebuild.sh
  14. BIN
      nilmbuntu.png
  15. +7
    -3
      run.sh
  16. BIN
      splash.png

+ 32
- 24
README View File

@@ -1,33 +1,41 @@
This live CD is customized by roughly following the guidelines at:
https://help.ubuntu.com/community/LiveCDCustomization
https://help.ubuntu.com/community/LiveCDCustomizationFromScratch
https://help.ubuntu.com/community/MakeALiveCD/DVD/BootableFlashFromHarddiskInstall
plus a lot of other info, like ubiquity and casper source code, and
additions for making the resulting iso EFI bootable.
Tools to customize an Ubuntu Live CD. Many parts of this will end up
being version-specific. The host machine must be running systemd,
since the image is executed inside a systemd-nspawn container.

extractiso.sh
extract the original iso
Set up host machine, which must be running a recent systemd:

customize.sh
apply customizations
sudo apt-get install squashfs-tools xorriso isolinux systemd-container iptables
sudo systemctl start systemd-networkd

enter.sh
chroot into the install (for optional manual customization)
Choose a config:

buildiso.sh
build the new iso
export BUILD_CONFIG=16.04

run.sh -c
boot the iso in qemu with a blank hdd, to test the install
Download and extract the original ISO:

run.sh -d
boot the installed hdd in qemu
./extractiso.sh

cleanup.sh
remove everything but the isos
Apply customizations, both to the outer ISO and the inner image.
This will also run customizer-inner.sh inside a container:

./customize.sh

SHORT VERSION:
sudo apt-get install squashfs-tools xorriso syslinux
time sudo sh -c "env BUILD_CONFIG=12.10 ./full-rebuild.sh"
time sudo sh -c "env BUILD_CONFIG=13.04 ./full-rebuild.sh"
(Optional) Run and enter a shell in the container, for manual customization:

./enter.sh

Build the new ISO:

./buildiso.sh

Boot the ISO in QEMU with a blank HDD, to test and install:

./run.sh -c

Boot QEMU with the same HDD image as above, to test an "installed" system:

./run.sh -d

Remove everything but the ISOs:

./cleanup.sh

+ 50
- 26
buildiso.sh View File

@@ -1,56 +1,80 @@
#!/bin/bash

# build the iso
# Build the iso.
# Some parts of this (particularly the xorriso command lines) come from
# the ubuntu-cdimage project:
# http://bazaar.launchpad.net/~ubuntu-cdimage/debian-cd/ubuntu/

# make sure this was run as root
if [ $UID -ne 0 ] ; then
echo "Need to be root; trying sudo"
exec sudo env BUILD_CONFIG=$BUILD_CONFIG $0 "$@"
fi
. config || exit 0

set -x
set -e

if ! [ "$1" == "skip" ] ; then
INITRD=${FS}/initrd.img
VMLINUZ=$(readlink -f ${FS}/vmlinuz)
VMLINUZ_EFI=${VMLINUZ}.efi.signed

if ! [ "$1" == "justiso" ] ; then

# copy kernel if changed
if [ ${FS}/initrd.img -nt ${ISO}/casper/initrd.lz ] ; then
sudo cp ${FS}/vmlinuz ${ISO}/casper/vmlinuz
sudo sh -c "zcat ${FS}/initrd.img | lzma > ${ISO}/casper/initrd.lz"
fi

# manifests
sudo chmod +w ${ISO}/casper/filesystem.manifest
sudo chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
| sudo tee ${ISO}/casper/filesystem.manifest >/dev/null
if ! [ -r ${INITRD} -a -r ${VMLINUZ} -a -r ${VMLINUZ_EFI} ] ; then
set +x
echo "== Missing kernel and/or initrd. Either upgrade or reinstall"
echo "== the kernel inside the image to get the right files."
exit 1
fi

# squashfs
sudo rm -f ${ISO}/casper/filesystem.squashfs
sudo mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
printf $(sudo du -sx --block-size=1 ${FS} | cut -f1) \
| sudo tee ${ISO}/casper/filesystem.size
# Recompress initrd if changed
if [ ${INITRD} -nt ${ISO}/casper/initrd.lz ] ; then
sh -c "zcat ${INITRD} | lzma > ${ISO}/casper/initrd.lz"
fi
# Get kernel and signed EFI kernel
cp ${VMLINUZ} ${ISO}/casper/vmlinuz
cp ${VMLINUZ_EFI} ${ISO}/casper/vmlinuz.efi

# manifests
chmod +w ${ISO}/casper/filesystem.manifest
chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
| tee ${ISO}/casper/filesystem.manifest >/dev/null

# squashfs
rm -f ${ISO}/casper/filesystem.squashfs
mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
printf $(du -sx --block-size=1 ${FS} | cut -f1) \
| tee ${ISO}/casper/filesystem.size

fi

# md5sums
sudo rm -f md5sum.txt
sudo find ${ISO} -type f -print0 \
| sudo xargs -0 md5sum \
rm -f md5sum.txt
find ${ISO} -type f -print0 \
| xargs -0 md5sum \
| sed -e "s, ${ISO}, .," \
| grep -v isolinux/boot.cat \
| grep -v isolinux/isolinux.bin \
| grep -v md5sum.txt \
| sudo tee ${ISO}/md5sum.txt >/dev/null
| tee ${ISO}/md5sum.txt >/dev/null

sudo chown -R ${USER} ${ISO}
chown -R ${NONPRIV_UID} ${ISO}

# build CD
xorriso -as mkisofs \
-D -r -V "NilmDBuntu ${VERSION}" -cache-inodes -J -l \
-input-charset utf-8 -o ${OUTPUT} \
-joliet -full-iso9660-filenames \
-rational-rock \
-V "NILMbuntu ${VERSION}" \
-input-charset utf-8 -output ${OUTPUT} \
-b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
-boot-load-size 4 -boot-info-table \
-eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
-isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin \
-isohybrid-gpt-basdat -isohybrid-apm-hfsplus \
${ISO}

# fix iso for hybrid booting
isohybrid ${OUTPUT}

set +x
echo "Burn it with:"
echo " growisofs -dvd-compat -Z /dev/dvd=${OUTPUT}"


+ 0
- 5
cleanup.sh View File

@@ -3,9 +3,4 @@
# delete all temporary stuff
. config || exit 0

sudo umount ${FS}/dev/pts || true
sudo umount ${FS}/sys/kernel/security || true
sudo umount ${FS}/sys || true
sudo umount ${FS}/proc || true
sudo umount ${MNT} || true
sudo rm --one-file-system -rf ${FS} ${ISO} ${MNT} ${DISK}

+ 6
- 2
config View File

@@ -1,9 +1,13 @@
#!/bin/bash

if [ -z "$BUILD_CONFIG" ] ; then
echo "Set BUILD_CONFIG first"
exit 1
export BUILD_CONFIG=16.04
echo "No BUILD_CONFIG set -- assuming $BUILD_CONFIG"
fi

echo "Loading config file: config-$BUILD_CONFIG"
. config-$BUILD_CONFIG
# echo " Ubuntu version: ${VERSION}"
# echo " Input ISO: ${ISONAME}"
# echo " from: ${ORIGURL}"
# echo " Output ISO: ${OUTPUT}"

+ 0
- 17
config-12.10 View File

@@ -1,17 +0,0 @@
# output image
VERSION="12.10.2"
OUTPUT="nilmdbuntu-${VERSION}.iso"

# original ISOs
ISONAME="xubuntu-12.10-desktop-amd64.iso"
ORIG="tmp-12.10/$ISONAME"
ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/12.10/release/$ISONAME"

# work directories, temporary qemu HD image
FS="tmp-12.10/fs"
ISO="tmp-12.10/iso"
MNT="tmp-12.10/mnt"
DISK="tmp-12.10/disk.img"

# local user
USER=jim:jim

+ 0
- 17
config-13.04 View File

@@ -1,17 +0,0 @@
# output image
VERSION="13.04.2"
OUTPUT="nilmdbuntu-${VERSION}.iso"

# original ISOs
ISONAME="xubuntu-13.04-desktop-amd64.iso"
ORIG="tmp-13.04/$ISONAME"
ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/13.04/release/$ISONAME"

# work directories, temporary qemu HD image
FS="tmp-13.04/fs"
ISO="tmp-13.04/iso"
MNT="tmp-13.04/mnt"
DISK="tmp-13.04/disk.img"

# local user
USER=jim:jim

+ 0
- 17
config-13.10 View File

@@ -1,17 +0,0 @@
# output image
VERSION="13.10.2"
OUTPUT="nilmdbuntu-${VERSION}.iso"

# original ISOs
ISONAME="xubuntu-13.10-desktop-amd64.iso"
ORIG="tmp-13.10/$ISONAME"
ORIGURL="http://hex1a4.net/xubuntu-13.10-desktop-amd64.iso"

# work directories, temporary qemu HD image
FS="tmp-13.10/fs"
ISO="tmp-13.10/iso"
MNT="tmp-13.10/mnt"
DISK="tmp-13.10/disk.img"

# local user
USER=jim:jim

+ 19
- 0
config-16.04 View File

@@ -0,0 +1,19 @@
# -*- sh -*-

# output image
VERSION="16.04"
OUTPUT="nilmbuntu-${VERSION}.iso"

# original ISO
ISONAME="xubuntu-${VERSION}-desktop-amd64.iso"
ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/${VERSION}/release/${ISONAME}"

# work directories, temporary qemu HD image, etc. Shouldn't need to change these.
ORIG="tmp-${VERSION}/$ISONAME"
FS="tmp-${VERSION}/fs"
ISO="tmp-${VERSION}/iso"
MNT="tmp-${VERSION}/mnt"
DISK="tmp-${VERSION}/disk.img"

# non-privileged user, so we don't need to do everything as root
NONPRIV_UID=${SUDO_UID:-$UID}

+ 224
- 297
customize-inner.sh View File

@@ -11,7 +11,9 @@ set -x
try_install() {
# try to install packages, but ignore failure
for pkg in "$@"; do
apt-get -y install "$pkg" || true
if ! apt-get -y install "$pkg" ; then
echo ---- WARNING: Failed to install package: $pkg
fi
done
}

@@ -19,35 +21,23 @@ try_install() {
cat >/etc/casper.conf <<"EOF"
export USERNAME="ubuntu"
export USERFULLNAME="Live session user"
export HOST="nilmdb"
export HOST="nilmbuntu"
export BUILD_SYSTEM="Ubuntu"
export FLAVOUR="NilmDBuntu"
export FLAVOUR="NILMbuntu"
EOF

# In 13.10, this is needed to upgrade the "whoopsie" package; bug #1272269
grep '13[.]10' /etc/issue && ln -sf /lib/init/upstart-job /etc/init.d/whoopsie

# Upgrade packages, remove old kernels
# Upgrade packages and remove old kernels
apt-get update
# in 13.04, doing upgrade & dist-upgrade together tries to install 2 kernels
# at the same time, which breaks for some reason. Also, try the upgrade
# multiple times since that can help
apt-get -y upgrade || apt-get -y upgrade || true
apt-get -y dist-upgrade || apt-get -y dist-upgrade || true
apt-get -y dist-upgrade
apt-get -y --purge autoremove
for VER in $(ls --sort=version /lib/modules/ | head -n -1) ; do
apt-get -y --purge remove ".*$VER.*"
apt-get -y --purge remove "linux-.*$VER"
done

# Disable upgrade popups
sed -i -s -e 's/Prompt=normal/Prompt=never/g' \
sed -i -s -e 's/Prompt=.*/Prompt=never/g' \
/etc/update-manager/release-upgrades || true

# some stuff we need from Ubuntu
try_install \
wbritish \
thunderbird-locale-en-us

# Set up & install postfix for local mail delivery
debconf-set-selections <<"EOF"
postfix postfix/mailname string localdomain
@@ -55,8 +45,9 @@ postfix postfix/main_mailer_type select Local only
EOF
apt-get -y install postfix

# install nilmdb things
# Required packages
apt-get -y install \
python3 \
python2.7 \
python2.7-dev \
python-setuptools \
@@ -80,301 +71,237 @@ apt-get -y install \
python-scipy \
python-daemon

# install other useful but optional stuff
# Install other useful but optional stuff
try_install \
emacs-goodies-el \
emacs23-nox \
octave \
octave-signal \
octave-missing-functions \
gnuplot \
curl \
devscripts \
dlocate \
emacs \
emacs-goodies-el \
gcc-arm-none-eabi \
gdb-arm-none-eabi \
gddrescue \
gnuplot \
help2man \
luatex \
pgf \
ipython \
libnewlib-arm-none-eabi \
libstdc++-arm-none-eabi-newlib \
texlive \
mailutils \
moreutils \
ntfsprogs \
subversion \
dlocate \
ack-grep \
mutt \
python-matplotlib \
ipython \
openvpn \
network-manager-openvpn-gnome \
octave \
octave-missing-functions \
octave-signal \
openocd \
openssl \
tcpdump \
openvpn \
python-matplotlib \
screen \
devscripts \
mailutils

# required
apt-get -y install \
openssh-server
silversearcher-ag \
subversion \
tcpdump \
zip

# Set up timezone to America/New_York for the live CD
echo America/New_York > /etc/timezone
ln -sf /usr/share/zoneinfo/America/New_York /etc/localtime
dpkg-reconfigure -f noninteractive tzdata

# Create nilmdb user to run the database
adduser --system --group --shell /bin/bash --disabled-password nilmdb
cp -rv /etc/skel/.??* /home/nilmdb
chown -R nilmdb:nilmdb /home/nilmdb

# Create WSGI scripts
cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
import nilmdb.server
application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
EOF
cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
import nilmrun.server
application = nilmrun.server.wsgi_application("/nilmrun")
EOF

#### Edit apache config

# Create apache config by hacking up the default one. Might be a better way
# to do this, and it'll probably break on new versions, but...

APACHE_VER=$(dpkg -s apache2 | grep ^Version | cut -d ' ' -f 2)
if dpkg --compare-versions $APACHE_VER ge 2.4 ; then
DEF=/etc/apache2/sites-available/000-default.conf
NEED_PERMISSIONS=1
else
DEF=/etc/apache2/sites-available/default
NEED_PERMISSIONS=0
fi

# Cut out any existing NilmDB stuff
perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig

# Copy everything up to the first </VirtualHost> line
perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF

# Add the NilmDB config
cat >>$DEF <<"EOF"
## NilmDB start
WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
<Location /nilmdb>
WSGIProcessGroup nilmdb-procgroup
WSGIApplicationGroup nilmdb-appgroup
</Location>

WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
<Location /nilmrun>
WSGIProcessGroup nilmrun-procgroup
WSGIApplicationGroup nilmrun-appgroup
</Location>
EOF
if [ $NEED_PERMISSIONS == 1 ] ; then
cat >>$DEF <<"EOF"

<Directory /home/nilmdb>
Options All
AllowOverride All
Require all granted
</Directory>
EOF
fi
cat >>$DEF <<"EOF"
## NilmDB end
EOF

# Copy everything including and after the first </VirtualHost> line
perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF

#### Done editing apache config

# Create nilmdb capture, processing, and cleanup files
cat > /home/nilmdb/capture.sh <<"EOF"
#!/bin/bash -e

# Don't run capture if we're running off a live CD
if grep -q boot=casper /proc/cmdline ; then
echo "Skipping capture, because this is a live CD."
exit 0
fi

echo "Starting capture in background..."
nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
"ethstream -a 192.168.1.209 -n 6 -r 8000" \
"nilm-insert -m 10 -r 8000 --live /data/raw"
EOF
cat > /home/nilmdb/process.sh <<"EOF"
#!/bin/bash -e

# Ensure only one copy of this code runs at a time:
LOCKFILE="/tmp/nilmdb-process.lock"
exec 99>"$LOCKFILE"
flock -n -x 99 || exit 0
trap 'rm -f "$LOCKFILE"' 0

nilm-sinefit -c 4 /data/raw /data/sinefit
nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
nilm-decimate-auto /data/raw "/data/prep*"
nilm-cleanup --yes /home/nilmdb/cleanup.cfg
EOF
cat > /home/nilmdb/cleanup.cfg <<"EOF"
[/data/prep-*]
keep = 1y

[/data/raw]
keep = 2w

[/data/sinefit]
keep = 1y
decimated = false
EOF

# Set up crontab
cat > /home/nilmdb/crontab <<"EOF"
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

# Run capture and processing scripts every 5 minutes
*/5 * * * * chronic /home/nilmdb/capture.sh
*/5 * * * * chronic /home/nilmdb/process.sh

# Try to run nilmdb-fsck on boot. It should hopefully run before
# apache opens the database, and apache will return errors to clients
# until nilmdb-fsck is done.
@reboot chronic nilmdb-fsck --fix --no-data /home/nilmdb/db
EOF
crontab -u nilmdb /home/nilmdb/crontab

# Fix permissions
chown -R nilmdb:nilmdb /home/nilmdb
chmod +x /home/nilmdb/{capture,process}.sh

# Fetch and build everything. Put it in the nilmdb dir
echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
GIT=https://git.jim.sh/jim/lees
rm -rf /home/nilmdb/git
mkdir /home/nilmdb/git
chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
REPOS="nilmdb nilmtools nilmrun ethstream"

# check it out as nilmdb, so the .netrc gets used
for repo in $REPOS; do
sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
done

# build as root, because we need to do that for the install
for repo in $REPOS; do
make -C /home/nilmdb/git/$repo install
done

# fix up all permissions in git dir, so nilmdb user can play with it later
chown -R nilmdb:nilmdb /home/nilmdb/git

# Create the initial database and streams by running the standalone
# server as nilmdb, making the right nilmtool calls, and killing it.
sudo -i -u nilmdb nilmdb-server -a 127.0.0.1 -p 18646 -d /home/nilmdb/db &
SERVERPID=$!
trap "kill -9 $SERVERPID" 0
for i in $(seq 1 120) ; do
sleep 1
echo waiting for nilmdb to start $i
if nilmtool -u http://127.0.0.1:18646/ info ; then
break
fi
done
nilmtool -u http://127.0.0.1:18646/ destroy -R "/data/*" || true
nilmtool -u http://127.0.0.1:18646/ create /data/raw uint16_6
nilmtool -u http://127.0.0.1:18646/ create /data/sinefit float32_3
nilmtool -u http://127.0.0.1:18646/ create /data/prep-a float32_8
nilmtool -u http://127.0.0.1:18646/ create /data/prep-b float32_8
nilmtool -u http://127.0.0.1:18646/ create /data/prep-c float32_8
kill $SERVERPID
wait
trap "" 0

# Put some default desktop shortcuts in place
DESKTOP=/etc/skel/Desktop
mkdir -p $DESKTOP
cp /usr/share/applications/exo-terminal-emulator.desktop $DESKTOP || true
cp /usr/share/applications/exo-web-browser.desktop $DESKTOP || true
chmod +x $DESKTOP/* # needs to be executable for 13.04+

# XFCE / theme customizations
if [ -d /usr/share/themes/Clearlooks ] ; then
cat > /usr/share/gconf/defaults/88_nilmdbuntu-gtk-theme <<"EOF"
/desktop/gnome/interface/gtk_theme "Clearlooks"
EOF
fi
if [ -d /usr/share/icons/elementary-xfce ] ; then
cat > /usr/share/gconf/defaults/88_nilmdbuntu-icon-theme <<"EOF"
/desktop/gnome/interface/icon_theme "elementary-xfce"
EOF
fi
update-gconf-defaults

chmod +x $DESKTOP/*

# Custom background image (which was already copied in by customize.sh)
XML=/etc/xdg/xdg-xubuntu/xfce4/xfconf/xfce-perchannel-xml
BG=/usr/share/xfce4/backdrops
mkdir -p $XML
cat >$XML/xfce4-desktop.xml <<"EOF"
<?xml version="1.0" encoding="UTF-8"?>

<channel name="xfce4-desktop" version="1.0">
<property name="desktop-icons" type="empty">
<property name="style" type="int" value="2"/>
<property name="file-icons" type="empty">
<property name="show-home" type="bool" value="true"/>
<property name="show-filesystem" type="bool" value="true"/>
<property name="show-removable" type="bool" value="true"/>
<property name="show-trash" type="bool" value="true"/>
</property>
</property>
<property name="backdrop" type="empty">
<property name="screen0" type="empty">
<property name="monitor0" type="empty">
<property name="image-path" type="string"
value="/usr/share/xfce4/backdrops/nilmdbuntu.png"/>
<property name="image-show" type="bool" value="true"/>
<property name="image-style" type="int" value="4"/>
<property name="color-style" type="int" value="0"/>
<property name="color1" type="array">
<value type="uint" value="0"/>
<value type="uint" value="0"/>
<value type="uint" value="0"/>
<value type="uint" value="65535"/>
</property>
</property>
<property name="monitor1" type="empty">
<property name="image-path" type="string"
value="/usr/share/xfce4/backdrops/nilmdbuntu.png"/>
<property name="image-show" type="bool" value="true"/>
<property name="image-style" type="int" value="4"/>
<property name="color-style" type="int" value="0"/>
<property name="color1" type="array">
<value type="uint" value="0"/>
<value type="uint" value="0"/>
<value type="uint" value="0"/>
<value type="uint" value="65535"/>
</property>
</property>
</property>
</property>
</channel>
EOF
sed -i -s -e 's/Greybird/Default/g' $XML/xfwm4.xml || true
sed -i -s -e 's/Greybird/Clearlooks/g' $XML/xsettings.xml || true
sed -i -s -e \
's/elementary-xfce-dark(er)?/elementary-xfce/g' $XML/xsettings.xml || true

# Firefox defaults
cat >/etc/firefox/syspref.js <<"EOF"
pref("browser.startup.homepage", "http://nilmdb.com/");
EOF
cat >/etc/xul-ext/homepage.properties <<"EOF"
browser.startup.homepage=http://nilmdb.com/
EOF
cat >/etc/xul-ext/ubufox.js <<"EOF"
pref("browser.startup.homepage", "file:/etc/xul-ext/homepage.properties");
sed -i -s -e 's/xubuntu-wallpaper.png/nilmbuntu.png/g' $XML/xfce4-desktop.xml

# Configure Firefox with our default homepage
HOMEPAGE="http://www.wattsworth.net/"
cat >/etc/firefox/syspref.js <<EOF
pref("browser.startup.homepage", "${HOMEPAGE}");
pref("extensions.ubufox@ubuntu.com.custom_homepage", "${HOMEPAGE}");
pref("browser.startup.homepage_override.mstone", "ignore");
EOF

# NilmDB setup below...

# # Create nilmdb user to run the database
# adduser --system --group --shell /bin/bash --disabled-password nilmdb
# cp -rv /etc/skel/.??* /home/nilmdb
# chown -R nilmdb:nilmdb /home/nilmdb

# # Create WSGI scripts
# cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
# import nilmdb.server
# application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
# EOF
# cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
# import nilmrun.server
# application = nilmrun.server.wsgi_application("/nilmrun")
# EOF

# #### Edit apache config

# # Create apache config by hacking up the default one. Might be a better way
# # to do this, and it'll probably break on new versions, but...

# APACHE_VER=$(dpkg -s apache2 | grep ^Version | cut -d ' ' -f 2)
# if dpkg --compare-versions $APACHE_VER ge 2.4 ; then
# DEF=/etc/apache2/sites-available/000-default.conf
# NEED_PERMISSIONS=1
# else
# DEF=/etc/apache2/sites-available/default
# NEED_PERMISSIONS=0
# fi

# # Cut out any existing NilmDB stuff
# perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig

# # Copy everything up to the first </VirtualHost> line
# perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF

# # Add the NilmDB config
# cat >>$DEF <<"EOF"
# ## NilmDB start
# WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
# WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
# <Location /nilmdb>
# WSGIProcessGroup nilmdb-procgroup
# WSGIApplicationGroup nilmdb-appgroup
# </Location>

# WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
# WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
# <Location /nilmrun>
# WSGIProcessGroup nilmrun-procgroup
# WSGIApplicationGroup nilmrun-appgroup
# </Location>
# EOF
# if [ $NEED_PERMISSIONS == 1 ] ; then
# cat >>$DEF <<"EOF"

# <Directory /home/nilmdb>
# Options All
# AllowOverride All
# Require all granted
# </Directory>
# EOF
# fi
# cat >>$DEF <<"EOF"
# ## NilmDB end
# EOF

# # Copy everything including and after the first </VirtualHost> line
# perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF

# #### Done editing apache config

# # Create nilmdb capture, processing, and cleanup files
# cat > /home/nilmdb/capture.sh <<"EOF"
# #!/bin/bash -e

# # Don't run capture if we're running off a live CD
# if grep -q boot=casper /proc/cmdline ; then
# echo "Skipping capture, because this is a live CD."
# exit 0
# fi

# echo "Starting capture in background..."
# nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
# "ethstream -a 192.168.1.209 -n 6 -r 8000" \
# "nilm-insert -m 10 -r 8000 --live /data/raw"
# EOF
# cat > /home/nilmdb/process.sh <<"EOF"
# #!/bin/bash -e

# # Ensure only one copy of this code runs at a time:
# LOCKFILE="/tmp/nilmdb-process.lock"
# exec 99>"$LOCKFILE"
# flock -n -x 99 || exit 0
# trap 'rm -f "$LOCKFILE"' 0

# nilm-sinefit -c 4 /data/raw /data/sinefit
# nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
# nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
# nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
# nilm-decimate-auto /data/raw "/data/prep*"
# nilm-cleanup --yes /home/nilmdb/cleanup.cfg
# EOF
# cat > /home/nilmdb/cleanup.cfg <<"EOF"
# [/data/prep-*]
# keep = 1y

# [/data/raw]
# keep = 2w

# [/data/sinefit]
# keep = 1y
# decimated = false
# EOF

# # Set up crontab
# cat > /home/nilmdb/crontab <<"EOF"
# SHELL=/bin/bash
# PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

# # Run capture and processing scripts every 5 minutes
# */5 * * * * chronic /home/nilmdb/capture.sh
# */5 * * * * chronic /home/nilmdb/process.sh

# # Try to run nilmdb-fsck on boot. It should hopefully run before
# # apache opens the database, and apache will return errors to clients
# # until nilmdb-fsck is done.
# @reboot chronic nilmdb-fsck --fix --no-data /home/nilmdb/db
# EOF
# crontab -u nilmdb /home/nilmdb/crontab

# # Fix permissions
# chown -R nilmdb:nilmdb /home/nilmdb
# chmod +x /home/nilmdb/{capture,process}.sh

# # Fetch and build everything. Put it in the nilmdb dir
# echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
# GIT=https://git.jim.sh/jim/lees
# rm -rf /home/nilmdb/git
# mkdir /home/nilmdb/git
# chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
# REPOS="nilmdb nilmtools nilmrun ethstream"

# # check it out as nilmdb, so the .netrc gets used
# for repo in $REPOS; do
# sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
# done

# # build as root, because we need to do that for the install
# for repo in $REPOS; do
# make -C /home/nilmdb/git/$repo install
# done

# # fix up all permissions in git dir, so nilmdb user can play with it later
# chown -R nilmdb:nilmdb /home/nilmdb/git

# # Create the initial database and streams by running the standalone
# # server as nilmdb, making the right nilmtool calls, and killing it.
# sudo -i -u nilmdb nilmdb-server -a 127.0.0.1 -p 18646 -d /home/nilmdb/db &
# SERVERPID=$!
# trap "kill -9 $SERVERPID" 0
# for i in $(seq 1 120) ; do
# sleep 1
# echo waiting for nilmdb to start $i
# if nilmtool -u http://127.0.0.1:18646/ info ; then
# break
# fi
# done
# nilmtool -u http://127.0.0.1:18646/ destroy -R "/data/*" || true
# nilmtool -u http://127.0.0.1:18646/ create /data/raw uint16_6
# nilmtool -u http://127.0.0.1:18646/ create /data/sinefit float32_3
# nilmtool -u http://127.0.0.1:18646/ create /data/prep-a float32_8
# nilmtool -u http://127.0.0.1:18646/ create /data/prep-b float32_8
# nilmtool -u http://127.0.0.1:18646/ create /data/prep-c float32_8
# kill $SERVERPID
# wait
# trap "" 0

+ 13
- 12
customize.sh View File

@@ -6,16 +6,16 @@ set -x

# Customize the outer ISO image
sed -i -s -e \
"s/DISKNAME.*/DISKNAME NilmDBuntu $VERSION by Jim Paris/" \
"s/DISKNAME.*/DISKNAME NILMbuntu $VERSION/" \
${ISO}/README.diskdefines

# The .disk/info file is important -- it's used by ubiquity to extract
# out the distro name in dialogs, and I think casper might use it too
echo "NilmDBuntu $VERSION by Jim Paris" > ${ISO}/.disk/info
echo "NILMbuntu $VERSION" > ${ISO}/.disk/info

# Set up preseed file
cp ${ISO}/preseed/xubuntu.seed ${ISO}/preseed/nilmdbuntu.seed
cat >> ${ISO}/preseed/nilmdbuntu.seed <<"EOF"
cp ${ISO}/preseed/xubuntu.seed ${ISO}/preseed/nilmbuntu.seed
cat >> ${ISO}/preseed/nilmbuntu.seed <<"EOF"
# Commands to run on successful install:
# - Regenerate SSH host keys
# - Remove "quiet splash" from grub command line
@@ -35,7 +35,8 @@ ubiquity ubiquity/success_command string \
umount /target/$i; done; \
echo "success_command done" >>/target/var/log/installer/postinst.log

# Default user. Ubiquity should let them change this
# Default user. Ubiquity should let them change this.
# (To skip the dialog completely, maybe set a password?)
d-i passwd/user-fullname string NILM User
d-i passwd/username string nilm
#d-i passwd/user-password password nilm
@@ -45,24 +46,24 @@ d-i passwd/auto-login boolean true
EOF

# Set up isolinux how we want by editing its config
TRY="Boot ^NilmDBuntu ${VERSION}"
TRY="Boot ^NILMbuntu ${VERSION}"
cp splash.png ${ISO}/isolinux/splash.png
perl -n -i \
-e '$n=1 if /^label/; $n=2 if /^label live-install/; next if $n==2;' \
-e "s/menu label.*(Try|NilmDBuntu).*/menu label $TRY/g;" \
-e "s,preseed/.*[.]seed,preseed/nilmdbuntu.seed,g;" \
-e "s/menu label.*(Try|NILM).*/menu label $TRY/g;" \
-e "s,preseed/.*[.]seed,preseed/nilmbuntu.seed,g;" \
-e "s/ quiet splash//g;" \
-e "print;" \
${ISO}/isolinux/txt.cfg
sed -i -s -e "s/^ui gfxboot/# ui gfxboot/g;" ${ISO}/isolinux/isolinux.cfg

# Set up grub similarly
TRY="Boot NilmDBuntu ${VERSION}"
TRY="Boot NILMbuntu ${VERSION}"
perl -n -i \
-e "next if /menuentry \"Install/../^}$/;" \
-e "next if /menuentry \"OEM install/../^}$/;" \
-e "s/menuentry \"(Try|NilmDBuntu).*\" {/menuentry \"${TRY}\" {/g;" \
-e "s,preseed/.*[.]seed,preseed/nilmdbuntu.seed,g;" \
-e "s/menuentry \"(Try|NILM).*\" \{/menuentry \"${TRY}\" {/g;" \
-e "s,preseed/.*[.]seed,preseed/nilmbuntu.seed,g;" \
-e "s/ quiet splash//g;" \
-e "print;" \
${ISO}/boot/grub/{grub,loopback}.cfg
@@ -70,7 +71,7 @@ perl -n -i \
if ! [ "$1" == "skip" ] ; then

# Run the customize-inner.sh script inside the chroot
sudo cp nilmdbuntu.png ${FS}/usr/share/xfce4/backdrops/nilmdbuntu.png
sudo cp nilmbuntu.png ${FS}/usr/share/xfce4/backdrops/nilmbuntu.png
sudo cp customize-inner.sh ${FS}/root/customize-inner.sh
sudo chmod +x ${FS}/root/customize-inner.sh
./enter.sh "cd /root ; ./customize-inner.sh"


+ 116
- 37
enter.sh View File

@@ -5,20 +5,91 @@ if [ $UID -ne 0 ] ; then
echo "Need to be root; trying sudo"
exec sudo env BUILD_CONFIG=$BUILD_CONFIG $0 "$@"
fi

# enter the chroot and run the command (if supplied) or a shell
. config || exit 0

# Spawn a systemd container that boots the machine, then run the given
# command. We can't execute systemd-nspawn directly, because that
# only allows us to either boot the machine, or run a command (not
# both). Instead let's execute systemd-nspawn in a transient systemd
# unit, then enter it using machinectl.

UNIT=nilmbuntu-run-$VERSION
MACH=nilmbuntu-$VERSION

setup_networking() {
# We use a virtual ethernet adapter -- this requires that
# systemd-networkd is installed and running on the host.
if ! systemctl is-active systemd-networkd ; then
echo "Starting systemd-networkd"
systemctl start systemd-networkd
fi

# However, the current systemd-networkd from Debian is broken and
# won't enable masquerading -- so do it manually
IFACE=$(ip -4 route list default | head -1 | awk '{print $5}')
echo 1 > /proc/sys/net/ipv4/ip_forward
iptables -t nat -D POSTROUTING -o $IFACE -j MASQUERADE >/dev/null || true
iptables -t nat -A POSTROUTING -o $IFACE -j MASQUERADE
}

kill_container() {
# Kill any running container
if systemctl --quiet is-active $UNIT ; then
echo "Stopping container..."
# We could use "machinectl terminate", but that will wait
# for a clean shutdown or timeout; we don't need a clean
# shutdown, so send a SIGTERM twice to get systemd-nspawn
# to terminate pretty quickly
systemctl kill $UNIT
sleep 2
systemctl kill $UNIT

# Then wait for it to really stop
systemctl stop $UNIT
fi

# If systemd-nspawn returned with a failure code,
# the transient service unit file will stick around,
# so make sure we clear that.
if systemctl --quiet is-failed $UNIT ; then
systemctl reset-failed $UNIT
fi
}

start_container() {
kill_container
echo "Starting container..."
systemd-run --unit=$UNIT systemd-nspawn \
--quiet \
--keep-unit \
--boot \
--network-veth \
--directory $(realpath $FS) \
--machine $MACH
echo "Waiting..."
while ! env SYSTEMD_LOG_LEVEL=0 machinectl shell $MACH /bin/true ; do
sleep 0.1
done
}

FAILED=0
run() {
# Run a command inside the container
echo "+" "$1"
chroot ${FS} env -i \
HOME=/root \
PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
TERM=$TERM \
IN_CHROOT=1 \
bash -c "$1"
RET=$?

# machinectl doesn't propagate return codes, so append something
# to the command that saves the result of what we ran.
echo "99" > $FS/jim-cmd-result
CMD="$1 ; echo \$? > /jim-cmd-result"

# Run it
env SYSTEMD_LOG_LEVEL=notice machinectl \
shell $MACH /usr/bin/env IN_CHROOT=1 \
bash -c "$CMD"

# Check result
RET=$(cat $FS/jim-cmd-result)
rm -f $FS/jim-cmd-result
if [ $RET -ne 0 ] && [ "$1" != "exec bash" ] ; then
printf "%s\n" "----------- WARNING: failed with exit code $RET"
FAILED=$RET
@@ -27,20 +98,19 @@ run() {
}

set -e
umount ${FS}/dev/pts || true
umount ${FS}/sys/kernel/security || true
umount ${FS}/sys || true
umount ${FS}/proc || true
mount -t proc none ${FS}/proc
mount -t sysfs none ${FS}/sys
mount -t devpts none ${FS}/dev/pts

run "echo 'nameserver 8.8.8.8' > /etc/resolv.conf"
run "dbus-uuidgen > /var/lib/dbus/machine-id"
run "dpkg-divert --local --rename --add /sbin/initctl"
run "ln -sf /bin/true /sbin/initctl"
run "dpkg-divert --local --rename --add /usr/sbin/update-grub"
run "ln -sf /bin/true /usr/sbin/update-grub"

setup_networking
start_container

run "resolvconf --disable-updates"
run "echo 'nameserver 8.8.8.8' > /run/resolvconf/resolv.conf"
run "hostnamectl --transient set-hostname nilmbuntu"

#run "dbus-uuidgen > /var/lib/dbus/machine-id"
#run "dpkg-divert --local --rename --add /sbin/initctl"
#run "ln -sf /bin/true /sbin/initctl"
#run "dpkg-divert --local --rename --add /usr/sbin/update-grub"
#run "ln -sf /bin/true /usr/sbin/update-grub"

set +e
if [ -z "$1" ] ; then
@@ -48,21 +118,30 @@ if [ -z "$1" ] ; then
else
run "$1"
fi

echo "Cleaning up..."

# Manually clean up some things that show up after booting an image
# and installing packages. This doesn't get everything, but what's
# left should be fine.
run "apt-get clean"
run "rm /sbin/initctl"
run "dpkg-divert --rename --remove /sbin/initctl"
run "rm /usr/sbin/update-grub"
run "dpkg-divert --rename --remove /usr/sbin/update-grub"
run "rm /var/lib/dbus/machine-id"
run "> /etc/resolv.conf"
run "rm -rf /tmp/* /tmp/.??* /root/.bash_history"

umount ${FS}/dev/pts
umount ${FS}/sys/kernel/security || true
umount ${FS}/sys
umount ${FS}/proc

echo "cleaned up"
run "> /etc/machine-id"
run "rm -f /core /boot/grub/grubenv"
run "rm -f /var/lib/systemd/random-seed"
run "rm -f /var/lib/ubuntu-drivers-common/last_gfx_boot"
run "rm -f /var/lib/NetworkManager/*"
run "rm -f /root/.bash_history"

#run "rm /sbin/initctl"
#run "dpkg-divert --rename --remove /sbin/initctl"
#run "rm /usr/sbin/update-grub"
#run "dpkg-divert --rename --remove /usr/sbin/update-grub"
#run "rm /var/lib/dbus/machine-id"
#run "> /etc/resolv.conf"

kill_container

echo "Done"
if [ $FAILED -ne 0 ] ; then
exit $FAILED
fi


+ 1
- 1
extractiso.sh View File

@@ -29,7 +29,7 @@ sudo mount -o loop,ro "$ORIG" ${MNT}
# copy data
sudo mkdir ${ISO}
sudo rsync --exclude=/casper/filesystem.squashfs -a ${MNT}/ ${ISO}
sudo chown -R ${USER} ${ISO}
sudo chown -R ${NONPRIV_UID} ${ISO}
chmod -R u+w ${ISO}

# copy squashfs


+ 2
- 3
full-rebuild.sh View File

@@ -11,13 +11,12 @@ fi

set -e

#rm -f ${OUTPUT}
./cleanup.sh
for d in ${ORIG} ${OUTPUT} ${DISK} ; do
mkdir -p $(dirname $d)
chown ${USER} $(dirname $d)
chown ${NONPRIV_UID} $(dirname $d)
done
./extractiso.sh
./customize.sh
./buildiso.sh
chown ${USER} ${ORIG} ${OUTPUT}
chown ${NONPRIV_UID} ${ORIG} ${OUTPUT}

BIN
nilmdbuntu.png → nilmbuntu.png View File

Before After
Width: 1920  |  Height: 1200  |  Size: 599 KiB Width: 1920  |  Height: 1200  |  Size: 589 KiB

+ 7
- 3
run.sh View File

@@ -45,17 +45,21 @@ esac
set -e
set -x

cfg=""
case $boot in
c)
echo "booting CD with empty disk"
rm -f ${DISK}
dd if=/dev/zero "of=${DISK}" bs=1M count=0 seek=10240
cfg="-cdrom ${iso} -hda ${DISK} -boot d"
cfg+=" -drive file=${iso},index=1,media=cdrom"
cfg+=" -drive file=${DISK},index=0,media=disk,format=raw"
cfg+=" -boot d"
;;
d)
echo "booting disk with no CD"
cfg="-hda ${DISK} -boot c"
cfg+=" -drive file=${DISK},index=0,media=disk,format=raw"
cfg+=" -boot c"
;;
esac

qemu-system-x86_64 -enable-kvm -m 1024 -vga vmware $cfg
qemu-system-x86_64 -enable-kvm -m 2048 -vga vmware $cfg

BIN
splash.png View File

Before After
Width: 640  |  Height: 480  |  Size: 1.4 KiB Width: 640  |  Height: 480  |  Size: 1.1 KiB

Loading…
Cancel
Save