nilmbuntu/customize-inner.sh

#!/bin/bash

if [ "$IN_CHROOT" != "1" ] ; then
    echo This is supposed to run inside the chroot, oops
    exit 1
fi

set -e
set -x

# Set up live username and hostname
cat >/etc/casper.conf <<"EOF"
export USERNAME="ubuntu"
export USERFULLNAME="Live session user"
export HOST="nilmdb"
export BUILD_SYSTEM="Ubuntu"
export FLAVOUR="NilmDBuntu"
EOF

# Upgrade packages, remove old kernels
apt-get update
apt-get -y dist-upgrade
apt-get -y --purge autoremove
for VER in $(ls --sort=version /lib/modules/ | head -n -1) ; do
    apt-get -y --purge remove ".*$VER.*"
done

# Disable upgrade popups
sed -i -s -e 's/Prompt=normal/Prompt=never/g' \
    /etc/update-manager/release-upgrades

# some stuff we need from Ubuntu
apt-get -y install \
    wbritish \
    thunderbird-locale-en-us

# install nilmdb things
apt-get -y install \
    python2.7 \
    python2.7-dev \
    python-setuptools \
    python-pip \
    cython \
    git \
    build-essential \
    python-cherrypy3 \
    python-decorator \
    python-simplejson \
    python-requests \
    python-dateutil \
    python-tz \
    python-progressbar \
    python-psutil \
    python-numpy \
    python-nose \
    python-coverage \
    apache2 \
    libapache2-mod-wsgi \
    python-scipy \
    python-daemon

# install other useful utilities
apt-get -y install \
    emacs-goodies-el \
    emacs23-nox \
    octave \
    octave-signal \
    octave-missing-functions \
    gnuplot \
    curl \
    gddrescue \
    help2man \
    luatex \
    pgf \
    moreutils \
    ntfsprogs \
    subversion \
    dlocate \
    ack-grep \
    mutt \
    openssh-server

# Set up timezone to America/New_York for the live CD
echo America/New_York > /etc/timezone
dpkg-reconfigure -f noninteractive tzdata

# Set up & install postfix for local mail delivery
debconf-set-selections <<"EOF"
postfix postfix/mailname string "localdomain"
postfix postfix/main_mailer_type select "Local only"
EOF
apt-get -y install postfix

# Create nilmdb user to run the database
adduser --system --group --shell /bin/bash --disabled-password nilmdb

# Create WSGI scripts
cat > /home/nilmdb/nilmdb.wsgi <<"EOF"
import nilmdb.server
application = nilmdb.server.wsgi_application("/home/nilmdb/db","/nilmdb")
EOF
cat > /home/nilmdb/nilmrun.wsgi <<"EOF"
import nilmrun.server
application = nilmrun.server.wsgi_application("/nilmrun")
EOF

# Create apache config by hacking up the default one.  Might be a better way
# to do this, and it'll probably break on apache 2.4, but...
DEF=/etc/apache2/sites-available/default
perl -ne 'print unless /## NilmDB start/../## NilmDB end/' $DEF > $DEF.orig
perl -ne 'print unless m-^[^#]*</VirtualHost>-..1' $DEF.orig > $DEF
cat >>$DEF <<"EOF"
	## NilmDB start
	WSGIScriptAlias /nilmdb /home/nilmdb/nilmdb.wsgi
	WSGIDaemonProcess nilmdb-procgroup threads=32 user=nilmdb group=nilmdb
	<Location /nilmdb>
	    WSGIProcessGroup nilmdb-procgroup
	    WSGIApplicationGroup nilmdb-appgroup
	</Location>

	WSGIScriptAlias /nilmrun /home/nilmdb/nilmrun.wsgi
	WSGIDaemonProcess nilmrun-procgroup threads=32 user=nilmdb group=nilmdb
	<Location /nilmrun>
	    WSGIProcessGroup nilmrun-procgroup
	    WSGIApplicationGroup nilmrun-appgroup
	</Location>
        ## NilmDB end
EOF
perl -ne 'print if m-^[^#]*</VirtualHost>-..1' $DEF.orig >> $DEF

# Create nilmdb capture, processing, and cleanup files
cat > /home/nilmdb/capture.sh <<"EOF"
#!/bin/bash -e

# Don't run capture if we're running off a live CD
if grep -q boot=casper /proc/cmdline ; then
    exit 0
fi

nilm-pipewatch --daemon --lock "/tmp/nilmdb-capture.lock" --timeout 30 \
    "ethstream -a 192.168.1.209 -n 6 -r 8000" \
    "nilm-insert -m 10 -r 8000 --live /data/raw"
EOF
cat > /home/nilmdb/process.sh <<"EOF"
#!/bin/bash -e

# Ensure only one copy of this code runs at a time:
LOCKFILE="/tmp/nilmdb-process.lock"
exec 99>"$LOCKFILE"
flock -n -x 99 || exit 0
trap 'rm -f "$LOCKFILE"' 0

nilm-sinefit -c 4 /data/raw /data/sinefit
nilm-prep -c 1 -r 0 /data/raw /data/sinefit /data/prep-a
nilm-prep -c 2 -r 120 /data/raw /data/sinefit /data/prep-b
nilm-prep -c 3 -r 240 /data/raw /data/sinefit /data/prep-c
nilm-decimate-auto /data/raw "/data/prep*"
nilm-cleanup --yes /home/nilmdb/cleanup.cfg
EOF
cat > /home/nilmdb/cleanup.cfg <<"EOF"
[/data/prep-*]
keep = 1y

[/data/raw]
keep = 2w

[/data/sinefit]
keep = 1y
decimated = false
EOF

# Set up crontab
cat > /home/nilmdb/crontab <<"EOF"
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

# Run capture and processing scripts every 5 minutes
*/5 * * * * chronic /home/nilmdb/capture.sh
*/5 * * * * chronic /home/nilmdb/process.sh

# Try to run nilmdb-fsck on boot.  It should hopefully run before
# apache opens the database, and apache will return errors to clients
# until nilmdb-fsck is done.
@reboot chronic nilmdb-fsck --fix --no-data /home/nilmdb/db
EOF
crontab -u nilmdb /home/nilmdb/crontab

# Fix permissions
chown -R nilmdb:nilmdb /home/nilmdb
chmod +x /home/nilmdb/{capture,process}.sh

# Fetch and build everything.  Put it in the nilmdb dir
echo "machine git.jim.sh login nilm password nilm" > /home/nilmdb/.netrc
GIT=https://git.jim.sh/jim/lees
rm -rf /home/nilmdb/git
mkdir /home/nilmdb/git
chown nilmdb:nilmdb /home/nilmdb/.netrc /home/nilmdb/git
REPOS="nilmdb nilmtools nilmrun ethstream"

# check it out as nilmdb, so the .netrc gets used
for repo in $REPOS; do
    sudo -i -u nilmdb git clone $GIT/$repo.git git/$repo
done

# build as root, because we need to do that for the install
for repo in $REPOS; do
    make -C /home/nilmdb/git/$repo install
done

# fix up all permissions in git dir, so nilmdb user can play with it later
chown -R nilmdb:nilmdb /home/nilmdb/git

# Create the initial database and streams by running the standalone
# server as nilmdb, making the right nilmtool calls, and killing it.
sudo -i -u nilmdb nilmdb-server -a 127.0.0.1 -p 18646 &
SERVER=$!
sleep 1
nilmtool -u http://127.0.0.1:18646/ destroy -R "/data/*" || true
nilmtool -u http://127.0.0.1:18646/ create /data/raw uint16_6
nilmtool -u http://127.0.0.1:18646/ create /data/sinefit float32_3
nilmtool -u http://127.0.0.1:18646/ create /data/prep-a float32_8
nilmtool -u http://127.0.0.1:18646/ create /data/prep-b float32_8
nilmtool -u http://127.0.0.1:18646/ create /data/prep-c float32_8
kill $!
wait