Browse Source

Finish first version of script

master
Jim Paris 9 months ago
parent
commit
a0a9b70fd7
3 changed files with 183 additions and 91 deletions
  1. +1
    -0
      Makefile
  2. +14
    -5
      README.md
  3. +168
    -86
      borg-setup.sh

+ 1
- 0
Makefile View File

@@ -1,4 +1,5 @@
test:
shellcheck -f gcc borg-setup.sh
rm -rf /tmp/test-borg
BORG_DIR=/tmp/test-borg ./borg-setup.sh
ls -al /tmp/test-borg


+ 14
- 5
README.md View File

@@ -1,10 +1,19 @@
# Design

On bucket:
- On bucket, we have a separate user account "jim-backups". Password
for this account is in bitwarden.

- New user account "jim-backups" with password in bitwarden
- Each client has 2 ssh keys in /home/jim-backups/.ssh/authorized_keys
(1) no password, for append-only operation
- Repository keys are repokeys, with passphrases saved on clients
and in bitwarden.

backup key in authorized_keys that allows append-only backups
- Each client has two SSH keys: one for append-only operation (no
pass) and one for read-write (password in bitwarden)

- Pruning requires the password and is a manual operation (run `sudo
/opt/borg/prune.sh`)

- Systemd timers start daily backups

# Usage

Run `borg-setup.sh` on a client

+ 168
- 86
borg-setup.sh View File

@@ -10,33 +10,31 @@ HOSTID="$(hostname -f)@$(python -c 'import uuid;print(uuid.getnode())')"

function error_handler() {
echo "Error at $1 line $2:"
echo -n '>>> ' ; cat $1 | tail -n +$2 | head -1
echo -n '>>> ' ; tail -n +"$2" < "$1" | head -1
echo "... exited with code $3"
exit $3
exit "$3"
}
trap 'error_handler ${BASH_SOURCE} ${LINENO} $?' ERR
set -o errexit
set -o errtrace
set -o nounset

if [ -e $BORG_DIR ]; then
if [ -e "$BORG_DIR" ]; then
echo "Error: BORG_DIR $BORG_DIR already exists; giving up"
exit 1
fi

# Make a temp dir to work in
mkdir $BORG_DIR
TMP=$(mktemp -d --tmpdir=$BORG_DIR)
mkdir "$BORG_DIR"
TMP=$(mktemp -d --tmpdir="$BORG_DIR")

# Install some cleanup handlers
cleanup()
{
set +o errexit
set +o errtrace
set +o nounset
trap - ERR
ssh -o ControlPath=$TMP/ssh-control -O exit x >/dev/null 2>&1
rm -rf -- $TMP
ssh -o ControlPath="$TMP"/ssh-control -O exit x >/dev/null 2>&1
rm -rf -- "$TMP"
}
cleanup_int()
{
@@ -47,35 +45,29 @@ cleanup_int()
trap cleanup 0
trap cleanup_int 1 2 15

error()
msg()
{
echo Error: "$@"
exit 1
}

log()
{
echo -ne "\033[1;33m===\033[0;33m" "$@"
echo -e "\033[0m"
}

notice()
{
echo -ne "\033[1;32m===\033[0;32m" "$@"
color="$1"
shift
echo -ne "\033[1;${color}m===\033[0;${color}m" "$@"
echo -e "\033[0m"
}
log(){ msg 33 "$@" ; }
notice() { msg 32 "$@" ; }
warn() { msg 31 "$@" ; }
error() { msg 31 "Error:" "$@" ; exit 1 ; }

# Install required packages
install_dependencies()
{
NEED=
check() {
command -v $1 >/dev/null || NEED+=" $2"
command -v "$1" >/dev/null || NEED+=" $2"
}
check borg borgbackup
if [ ! -z ${NEED:+x} ]; then
if [ -n "${NEED:+x}" ]; then
log "Need to install packages: $NEED"
apt install --no-upgrade $NEED
apt install --no-upgrade "$NEED"
fi
}

@@ -85,7 +77,7 @@ create_borg_wrapper()
BORG=${BORG_DIR}/borg.sh
BORG_REPO="ssh://${BACKUP_USER}@${BACKUP_HOST}/./${BACKUP_REPO}"

cat >$BORG <<EOF
cat >"$BORG" <<EOF
#!/bin/sh

export BORG_REPO=${BORG_REPO}
@@ -97,8 +89,8 @@ export BORG_CONFIG_DIR=${BORG_DIR}/config
export BORG_RSH="ssh -F ${BORG_DIR}/ssh/config"
exec borg "\$@"
EOF
chmod +x $BORG
if ! $BORG -h >/dev/null ; then
chmod +x "$BORG"
if ! "$BORG" -h >/dev/null ; then
error "Can't run the new borg wrapper; does borg work?"
fi

@@ -113,35 +105,35 @@ generate_keys()
{
PASS_SSH=$(print_random_key)
PASS_REPOKEY=$(print_random_key)
cat >${BORG_DIR}/print-passphrase <<EOF
cat >"${BORG_DIR}/print-passphrase" <<EOF
#!/bin/sh
echo $PASS_REPOKEY
EOF
chmod 700 ${BORG_DIR}/print-passphrase
chmod 700 "${BORG_DIR}/print-passphrase"
}

# Run a command on the remote host over an existing SSH tunnel
run_ssh_command()
{
ssh -o ControlPath=$TMP/ssh-control use-existing-control-tunnel "$@"
ssh -o ControlPath="$TMP"/ssh-control use-existing-control-tunnel "$@"
}

# Configure SSH key-based login
configure_ssh()
{
SSH=$BORG_DIR/ssh
mkdir $SSH
mkdir "$SSH"

# Create keys
log "Creating SSH keys"
ssh-keygen -N "" -t ecdsa \
-C backup-appendonly@$HOSTID -f $SSH/id_ecdsa_appendonly
-C "backup-appendonly@$HOSTID" -f "$SSH/id_ecdsa_appendonly"
ssh-keygen -N "$PASS_SSH" -t ecdsa \
-C backup@$HOSTID -f $SSH/id_ecdsa
-C "backup@$HOSTID" -f "$SSH/id_ecdsa"

# Create config snippets
log "Creating SSH config and wrapper script"
cat >> $SSH/config <<EOF
cat >> "$SSH/config" <<EOF
User $BACKUP_USER
ControlPath none
ServerAliveInterval 120
@@ -164,8 +156,8 @@ EOF
# Connect to backup host, using persistent control socket
log "Connecting to server"
log "Please enter password; look in Bitwarden for: ${BACKUP_USER}@${BACKUP_HOST}"
ssh -F $SSH/config -o BatchMode=no -o PubkeyAuthentication=no \
-o ControlMaster=yes -o ControlPath=$TMP/ssh-control \
ssh -F "$SSH/config" -o BatchMode=no -o PubkeyAuthentication=no \
-o ControlMaster=yes -o ControlPath="$TMP/ssh-control" \
-o StrictHostKeyChecking=accept-new \
-f backup sleep 600
if ! run_ssh_command true >/dev/null 2>&1 </dev/null ; then
@@ -189,13 +181,13 @@ EOF
run_ssh_command "mv $keys $backup; sed '/@$HOSTID\$/d' < $backup > $keys"
run_ssh_command "if cmp -s $backup $keys; then rm $backup ; fi"
run_ssh_command "cat >> .ssh/authorized_keys" <<EOF
command="$cmd --append-only",restrict $(cat $SSH/id_ecdsa_appendonly.pub)
command="$cmd",restrict $(cat $SSH/id_ecdsa.pub)
command="$cmd --append-only",restrict $(cat "$SSH/id_ecdsa_appendonly.pub")
command="$cmd",restrict $(cat "$SSH/id_ecdsa.pub")
EOF

# Test that everything worked
log "Testing SSH login with new key"
if ! ssh -F $SSH/config -T backup-appendonly borg --version </dev/null ; then
if ! ssh -F "$SSH/config" -T backup-appendonly borg --version </dev/null ; then
error "Logging in with a key failed -- is server set up correctly?"
fi
log "Remote connection OK!"
@@ -213,65 +205,149 @@ create_repo()
export_keys()
{
log "Exporting keys"
$BORG key export --paper '' ${BORG_DIR}/key.txt
chmod 600 ${BORG_DIR}/key.txt
cat >>${BORG_DIR}/key.txt <<EOF
$BORG key export --paper '' "${BORG_DIR}/key.txt"
chmod 600 "${BORG_DIR}/key.txt"
cat >>"${BORG_DIR}/key.txt" <<EOF

Repository: ${BORG_REPO}
Passphrase: ${PASS_REPOKEY}
EOF
}

# Create a script that can be run to make a new backup
create_backup_script()
# Create helper scripts to backup, prune, and mount
create_scripts()
{
SCRIPT=${BORG_DIR}/backup.sh
cat > "${BORG_DIR}/backup.sh" <<EOF
#!/bin/bash

BORG=$BORG_DIR/borg.sh
set -e

log "Creating backup script"
# Explicitly list a bunch of directories to back up, in case they come
# from different filesystems. If not, duplicates have no effect.
DIRS="/"
for DIR in /usr /var /home /boot /efi ; do
if [ -e $DIR ] ; then
DIRS="\$DIRS \$DIR"
fi
done

cat > $SCRIPT <<EOF
#!/bin/bash
# Allow dirs to be overridden
BORG_BACKUP_DIRS=\${BORG_BACKUP_DIRS:-\$DIRS}

# Generated by borg-setup
echo "Backing up: $DIRS"

\$BORG create \\
--verbose \\
--list \\
--filter E \\
--stats \\
--exclude-caches \\
--one-file-system \\
--checkpoint-interval 300 \\
--compression zstd,3 \\
::'{hostname}-{now:%Y%m%d-%H%M%S}' \\
\$BORG_BACKUP_DIRS

\$BORG check \\
--verbose \\
--last 10
EOF

cat > "${BORG_DIR}/prune.sh" <<EOF
#!/bin/bash

BORG=$BORG_DIR/borg.sh
set -e

echo "=== Need SSH key passphrase. Check Bitwarden for:"
echo "=== borg $(hostname)"
echo "=== read-write SSH key"
\$BORG prune \\
--rsh="ssh -F ${BORG_DIR}/ssh/config -i ${SSH}/id_ecdsa"
--verbose \\
--stats \\
--keep-within=7d \\
--keep-daily=14 \\
--keep-weekly=8 \\
--keep-monthly=-1
EOF

cat > "${BORG_DIR}/mount.sh" <<EOF
#!/bin/bash

BORG=$BORG_DIR/borg.sh
set -e

if [ -z \${DIRS:+x} ]; then
DIRS=/
# Include mountpoints that we care about, since we otherwise don't
# follow mounts
for DIR in /usr /var /home /boot ; do
if [ \$(stat -c%m \$DIR) = \$DIR ] ; then
DIRS="\$DIRS \$DIR"
fi
done
if [ -z "\$1" ] ; then
echo "Usage: \$0 <mountpoint>"
exit 1
fi

echo "Backing up ${DIRS}"
\$BORG mount \\
:: \$1

echo "Unmount with: fusermount -u \$1"

ionice -c 2 -n 6 \\
nice -n 10 \\
\$BORG create \\
--verbose \\
--list \\
--filter E \\
--stats \\
--exclude-caches \\
--one-file-system \\
--checkpoint-interval 300 \\
--compression zstd,3 \\
::'{hostname}-{now:%Y%m%d-%H%M%S}' \\
\$DIRS

\$BORG check --verbose

# Run manual commands like this:
# sudo $BORG_DIR/borg.sh list
EOF
chmod +x $SCRIPT
log "Script created: $SCRIPT"
}

configure_systemd()
{
TIMER=borg-backup.timer
SERVICE=borg-backup.service
TIMER_UNIT=${BORG_DIR}/${TIMER}
SERVICE_UNIT=${BORG_DIR}/${SERVICE}

log "Creating systemd files"

cat > "$TIMER_UNIT" <<EOF
[Unit]
Description=Borg backup to ${BACKUP_HOST}

[Timer]
OnCalendar=*-*-* 01:00:00
RandomizedDelaySec=1800
FixedRandomDelay=true
Persistent=true

[Install]
WantedBy=timers.target
EOF

cat >> "$SERVICE_UNIT" <<EOF
[Unit]
Description=Borg backup to ${BACKUP_HOST}

[Service]
Type=simple
ExecStart=${BORG_DIR}/backup.sh
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=6
EOF

log "Setting up systemd"
if (
ln -sfv "${TIMER_UNIT}" /etc/systemd/system &&
ln -sfv "${SERVICE_UNIT}" /etc/systemd/system &&
systemctl --no-ask-password daemon-reload &&
systemctl --no-ask-password enable ${TIMER} &&
systemctl --no-ask-password start ${TIMER}
); then
log "Backup timer installed:"
systemctl list-timers ${TIMER}
else
warn ""
warn "Systemd setup failed"
warn "Do something like this to configure automatic backups:"
echo " sudo ln -sfv \"${TIMER_UNIT}\" /etc/systemd/system &&"
echo " sudo ln -sfv \"${SERVICE_UNIT}\" /etc/systemd/system &&"
echo " sudo systemctl daemon-reload &&"
echo " sudo systemctl enable ${TIMER} &&"
echo " sudo systemctl start ${TIMER}"
warn ""
fi
}

log "Configuration:"
@@ -285,10 +361,10 @@ generate_keys
configure_ssh
create_repo
export_keys
create_backup_script
create_scripts
configure_systemd

log "All done"
notice ""
echo
notice "Add these two passwords to Bitwarden:"
notice ""
notice " Name: borg $(hostname)"
@@ -299,7 +375,13 @@ notice " Name: borg $(hostname)"
notice " Username: read-write ssh key"
notice " Password: $PASS_SSH"
notice ""
notice "Full repo key should be printed out: ${BORG_DIR}/key.txt"
notice "You should also print out the full repo key: ${BORG_DIR}/key.txt"
notice ""
notice "To run a manual backup:"
notice " sudo systemctl start borg-backup"
notice ""
notice "and to see logs:"
notice " journalctl -u borg-backup"
echo

#create_crontab_entry
echo "All done"

Loading…
Cancel
Save