Browse Source

Initial commit

tags/nilmdbuntu-1
Jim Paris 8 years ago
commit
918765b14d
10 changed files with 477 additions and 0 deletions
  1. +4
    -0
      .gitignore
  2. +30
    -0
      README
  3. +46
    -0
      buildiso.sh
  4. +7
    -0
      cleanup.sh
  5. +16
    -0
      config
  6. +180
    -0
      customize-inner.sh
  7. +38
    -0
      customize.sh
  8. +59
    -0
      enter.sh
  9. +36
    -0
      extractiso.sh
  10. +61
    -0
      run.sh

+ 4
- 0
.gitignore View File

@@ -0,0 +1,4 @@
fs
iso
mnt
*.iso

+ 30
- 0
README View File

@@ -0,0 +1,30 @@
This live CD is customized by roughly following the guidelines at:
https://
help.ubuntu.com/community/LiveCDCustomization
help.ubuntu.com/community/LiveCDCustomizationFromScratch
help.ubuntu.com/community/MakeALiveCD/DVD/BootableFlashFromHarddiskInstall
plus a lot of other info, like ubiquity and casper source code, and
additions for making the resulting iso EFI bootable.

Run things in generally this order:

extractiso.sh
extract the original iso

customize.sh
apply customizations

enter.sh
chroot into the install (for optional manual customization)

buildiso.sh
build the new iso

run.sh -c
boot the iso in qemu with a blank hdd, to test the install

run.sh -d
boot the installed hdd in qemu

cleanup.sh
remove everything but the isos

+ 46
- 0
buildiso.sh View File

@@ -0,0 +1,46 @@
#!/bin/bash

# build the iso
. config || exit 0

set -x
set -e

# copy kernel if changed
if [ ${FS}/initrd.img -nt ${ISO}/casper/initrd.lz ] ; then
sudo cp ${FS}/vmlinuz ${ISO}/casper/vmlinuz
sudo sh -c "zcat ${FS}/initrd.img | lzma > ${ISO}/casper/initrd.lz"
fi

# manifests
sudo chmod +w ${ISO}/casper/filesystem.manifest
sudo chroot ${FS} dpkg-query -W --showformat='${Package} ${Version}\n' \
| sudo tee ${ISO}/casper/filesystem.manifest >/dev/null

# squashfs
sudo rm -f ${ISO}/casper/filesystem.squashfs
sudo mksquashfs ${FS} ${ISO}/casper/filesystem.squashfs
printf $(sudo du -sx --block-size=1 ${FS} | cut -f1) \
| sudo tee ${ISO}/casper/filesystem.size

# md5sums
cd ${ISO}
sudo rm -f md5sum.txt
sudo find . -type f -print0 \
| sudo xargs -0 md5sum \
| grep -v isolinux/boot.cat \
| sudo tee md5sum.txt >/dev/null
cd ..

# build CD
cd ${ISO}
REALOUT=$(realpath ${OUTPUT})
sudo xorriso -as mkisofs \
-D -r -V "NilmDBuntu" -cache-inodes -J -l \
-input-charset utf-8 -o ${REALOUT} \
-b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
-boot-load-size 4 -boot-info-table \
-eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
.
cd ..
sudo chown jim:jim ${REALOUT}

+ 7
- 0
cleanup.sh View File

@@ -0,0 +1,7 @@
#!/bin/bash

# build the iso
. config || exit 0

sudo umount mnt || ture
sudo rm --one-file-system -rf ${FS} ${ISO} ${MNT} ${DISK}

+ 16
- 0
config View File

@@ -0,0 +1,16 @@
# output image
VERSION="12.10.0"
OUTPUT="nilmdbuntu-${VERSION}.iso"

# original ISOs
ORIG="xubuntu-12.10-desktop-amd64.iso"
ORIGURL="http://cdimage.ubuntu.com/xubuntu/releases/12.10/release/${ORIG}"

# work directories, temporary qemu HD image
FS="fs"
ISO="iso"
MNT="mnt"
DISK="disk.img"

# local user
USER=jim:jim

+ 180
- 0
customize-inner.sh View File

@@ -0,0 +1,180 @@
#!/bin/bash

if [ "$IN_CHROOT" != "1" ] ; then
echo This is supposed to run inside the chroot, oops
exit 1
fi

set -e
set -x

# Set up live username and hostname
cat >/etc/casper.conf <<"EOF"
export USERNAME="ubuntu"
export USERFULLNAME="Live session user"
export HOST="nilmdb"
export BUILD_SYSTEM="Ubuntu"
export FLAVOUR="NilmDBuntu"
EOF

# Upgrade packages, remove old kernels
apt-get update
apt-get -y dist-upgrade
apt-get -y --purge autoremove
for VER in $(ls --sort=version /lib/modules/ | head -n -1) ; do
apt-get -y --purge remove ".*$VER.*"
done

# install nilmdb things
apt-get -y install \
python2.7 \
python2.7-dev \
python-setuptools \
python-pip \
cython \
git \
build-essential \
python-cherrypy3 \
python-decorator \
python-simplejson \
python-requests \
python-dateutil \
python-tz \
python-progressbar \
python-psutil \
python-numpy \
python-nose \
python-coverage \
apache2 \
libapache2-mod-wsgi \
python-scipy \
python-daemon

# install other useful utilities
apt-get -y install \
emacs-goodies-el \
emacs23-nox \
octave \
octave-signal \
octave-missing-functions \
gnuplot \
curl \
gddrescue \
help2man \
luatex \
pgf \
moreutils \
ntfsprogs \
subversion \
dlocate \
ack-grep

# Set up & install postfix for local mail delivery
debconf-set-selections <<"EOF"
postfix postfix/mailname string "localdomain"
postfix postfix/main_mailer_type select "Local only"
EOF
apt-get -y install postfix

# Create nilmdb user to run the database
adduser --system --group --shell /bin/bash --disabled-password nilmdb

# Create WSGI scripts
cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
import nilmdb.server
application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
EOF
cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
import nilmrun.server
application = nilmrun.server.wsgi_application("/nilmrun")
EOF

# Create apache config by hacking up the default one. Might be a better way
# to do this, and it'll probably break on apache 2.4, but...
DEF=/etc/apache2/sites-available/default
perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig
perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF
cat >>$DEF <<"EOF"
## NilmDB start
WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
<Location /nilmdb>
WSGIProcessGroup nilmdb-procgroup
WSGIApplicationGroup nilmdb-appgroup
</Location>

WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
<Location /nilmrun>
WSGIProcessGroup nilmrun-procgroup
WSGIApplicationGroup nilmrun-appgroup
</Location>
## NilmDB end
EOF
perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF

# Create nilmdb capture, processing, and cleanup files
cat > /home/nilmdb/capture.sh <<"EOF"
#!/bin/bash -e

# Don't run capture if we're running off a live CD
if grep -q boot=casper /proc/cmdline ; then
exit 0
fi

nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
"ethstream -a 192.168.1.209 -n 6 -r 8000" \
"nilm-insert -m 10 -r 8000 --live /data/raw"
EOF
cat > /home/nilmdb/process.sh <<"EOF"
#!/bin/bash -e

# Ensure only one copy of this code runs at a time:
LOCKFILE="/tmp/nilmdb-process.lock"
exec 99>"$LOCKFILE"
flock -n -x 99 || exit 0
trap 'rm -f "$LOCKFILE"' 0

nilm-sinefit -c 4 /data/raw /data/sinefit
nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
nilm-decimate-auto /data/raw "/data/prep*"
nilm-cleanup --yes /home/nilmdb/cleanup.cfg
EOF
cat > /home/nilmdb/cleanup.cfg <<"EOF"
[/data/prep-*]
keep = 1y

[/data/raw]
keep = 2w

[/data/sinefit]
keep = 1y
decimated = false
EOF

# Fix permissions
chown -R nilmdb:nilmdb /home/nilmdb
chmod +x /home/nilmdb/{capture,process}.sh

# Fetch and build everything. Put it in the nilmdb dir
echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
GIT=https://git.jim.sh/jim/lees
rm -rf /home/nilmdb/git
mkdir /home/nilmdb/git
chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
REPOS="nilmdb nilmtools nilmrun ethstream"

# check it out as nilmdb, so the .netrc gets used
for repo in $REPOS; do
sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
done

# build as root, because we need to do that for the install
for repo in $REPOS; do
make -C /home/nilmdb/git/$repo install
done

# fix up all permissions in git dir, so nilmdb user can play with it later
chown -R nilmdb:nilmdb /home/nilmdb/git

+ 38
- 0
customize.sh View File

@@ -0,0 +1,38 @@
#!/bin/bash

. config || exit 0
set -e
set -x

# Customize the outer ISO image
perl -p -i -e \
"s/DISKNAME.*/DISKNAME NilmDBuntu $VERSION by Jim Paris/" \
${ISO}/README.diskdefines

# The .disk/info file is important -- it's used by ubiquity to extract
# out the distro name in dialogs, and I think casper might use it too
echo "NilmDBuntu $VERSION by Jim Paris" > ${ISO}/.disk/info

# Include openssh server at install time. Can't include it in the
# image, because then we'd share keys on all machines, I think.
cp ${ISO}/preseed/xubuntu.seed ${ISO}/preseed/nilmdbuntu.seed
cat >> ${ISO}/preseed/nilmdbuntu.seed <<EOF
d-i pkgsel/include string openssh-server
EOF

# Rename Xubuntu to NilmDBuntu in boot scripts; also changes
# preseed file to the custom one
perl -p -i \
-e "s/Xubuntu/NilmDBuntu/g;" \
-e "s/xubuntu/nilmdbuntu/g;" \
${ISO}/isolinux/txt.cfg ${ISO}/boot/grub/{grub,loopback}.cfg

# Remove quiet and splash from boot command lines. Easier than a
# custom plymouth theme and it's helpful.
perl -p -i -e "s/ quiet splash//g;" \
${ISO}/isolinux/txt.cfg ${ISO}/boot/grub/{grub,loopback}.cfg

# Run the customize-inner.sh script inside the chroot
sudo cp customize-inner.sh ${FS}/root/customize-inner.sh
sudo chmod +x ${FS}/root/customize-inner.sh
./enter.sh "cd /root ; ./customize-inner.sh"

+ 59
- 0
enter.sh View File

@@ -0,0 +1,59 @@
#!/bin/bash

# make sure this was run as root
if [ $UID -ne 0 ] ; then
echo "Need to be root; trying sudo"
exec sudo $0 "$@"
fi

# enter the chroot and run the command (if supplied) or a shell
. config || exit 0

run() {
echo "+" "$1"
chroot ${FS} env -i \
HOME=/root \
PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
TERM=$TERM \
IN_CHROOT=1 \
bash -c "$1"
RET=$?
if [ $RET -ne 0 ] && [ "$1" != "exec bash" ] ; then
printf "%s\n" "----------- WARNING: failed with exit code $RET"
sleep 5
fi
}

set -e
mount -t proc none ${FS}/proc
mount -t sysfs none ${FS}/sys
mount -t devpts none ${FS}/dev/pts

run "echo 'nameserver 8.8.8.8' > /etc/resolv.conf"
run "dbus-uuidgen > /var/lib/dbus/machine-id"
run "dpkg-divert --local --rename --add /sbin/initctl"
run "ln -s /bin/true /sbin/initctl"
run "dpkg-divert --local --rename --add /usr/sbin/update-grub"
run "ln -s /bin/true /usr/sbin/update-grub"

set +e
if [ -z "$1" ] ; then
run "exec bash"
else
run "$1"
fi
run "apt-get clean"
run "rm /sbin/initctl"
run "dpkg-divert --rename --remove /sbin/initctl"
run "rm /usr/sbin/update-grub"
run "dpkg-divert --rename --remove /usr/sbin/update-grub"
run "rm /var/lib/dbus/machine-id"
run "> /etc/resolv.conf"
run "rm -rf /tmp/* /tmp/.??* /root/.bash_history"

umount ${FS}/dev/pts
umount ${FS}/sys/kernel/security || true
umount ${FS}/sys
umount ${FS}/proc

echo "cleaned up"

+ 36
- 0
extractiso.sh View File

@@ -0,0 +1,36 @@
#!/bin/bash

# build the iso
. config || exit 0

if [ "$1" != "ok" ]; then
if [ -e ${ISO} ] || [ -e ${FS} ]; then
echo "remove \"${ISO}\" and \"${FS}\" dirs first,"
echo "or pass \"ok\" as an argument to remove them"
exit 0
fi
fi

set -x
set -e

# download it if it doesn't exist
if ! [ -e ${ORIG} ] ; then
wget -O "${ORIG}" "${ORIGURL}"
fi

# mount it
sudo umount ${MNT} || true
sudo rm -rf ${MNT} ${ISO} ${FS}
sudo mkdir ${MNT}
sudo mount -o loop,ro "$ORIG" ${MNT}

# copy data
sudo mkdir ${ISO}
sudo rsync --exclude=/casper/filesystem.squashfs -a ${MNT}/ ${ISO}
sudo umount ${MNT}
sudo chown -R ${USER} ${ISO}
chmod -R u+w ${ISO}

# copy squashfs
sudo unsquashfs -d ${FS} ${MNT}/casper/filesystem.squashfs

+ 61
- 0
run.sh View File

@@ -0,0 +1,61 @@
#!/bin/bash

# try to boot the ISO in qemu
. config || exit 0

usage() {
cat <<EOF
usage: $0 [-i iso] <-c | -d>

-c Make an empty disk image and boot from CD
-d Boot the disk image
-i Path to iso
EOF
exit 0
}

boot=""
iso="${OUTPUT}"

while getopts cdi: flag; do
case $flag in
c)
boot="c$boot"
;;
d)
boot="d$boot"
;;
i)
iso="$OPTARG"
;;
?)
usage
;;
esac
done

case $boot in
?)
;;
*)
usage
;;
esac

set -e
set -x

case $boot in
c)
echo "booting CD with empty disk"
rm -f disk.img
dd if=/dev/zero "of=${DISK}" bs=1M count=0 seek=10240
cfg="-cdrom ${iso} -hda ${DISK} -boot d"
;;
d)
echo "booting disk with no CD"
cfg="-hda ${DISK} -boot c"
;;
esac

qemu-system-x86_64 -enable-kvm -m 1024 -vga vmware $cfg

Loading…
Cancel
Save