Continue reworking towards local copy of borg, etc

.gitignore

@@ -1,2 +1,3 @@
-README.html
+*.html
+
 

Makefile

@@ -10,21 +10,26 @@ all:
 	@echo
 
 .PHONY: ctrl
-ctrl: test
+ctrl: test-setup
 
-.PHONY: test-lister
-test-lister: .venv
-	venv/bin/mypy lister.py
-	venv/bin/python lister.py --max-size 1GiB --one-file-system /tmp | grep -a 'bigf'
+.venv:
+	mkdir .venv
+	pipenv install --dev
 
-.PHONY: check
-check:
+.PHONY: test-backup
+test-backup: .venv
+	.venv/bin/mypy backup.py
+	./backup.py --max-size 1GiB --one-file-system /tmp | grep -a 'bigf'
+
+.PHONY: test-setup
+test-setup:
 	shellcheck -f gcc initial-setup.sh
-
-.PHONY: test
-test: check
 	rm -rf /tmp/test-borg
 	mkdir /tmp/test-borg
 	: "normally this would be a git clone, but we want the working tree..."
 	git ls-files -z | tar --null -T - -cf - | tar -C /tmp/test-borg -xvf -
 	/tmp/test-borg/initial-setup.sh
+
+.PHONY: clean
+clean:
+	rm -f README.html

Pipfile

@@ -5,9 +5,9 @@ name = "pypi"
 
 [packages]
 humanfriendly = "*"
-mypy = "*"
 
 [dev-packages]
+mypy = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3"

Pipfile.lock

@@ -1,11 +1,11 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "775048a9d9eea3ab29a1e53636271f45f9fe40ec250225818155d3eced6034e7"
+            "sha256": "4f504c785e3ed5b203a82a5f40516507f80a01b8d1d0ad5a905f139cafc41a51"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "3.9"
+            "python_version": "3"
         },
         "sources": [
             {
@@ -23,7 +23,9 @@
             ],
             "index": "pypi",
             "version": "==10.0"
-        },
+        }
+    },
+    "develop": {
         "mypy": {
             "hashes": [
                 "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9",
@@ -75,6 +77,5 @@
             ],
             "version": "==3.10.0.2"
         }
-    },
-    "develop": {}
+    }
 }

README.md

@@ -1,4 +1,72 @@
-# Design
+Initial setup
+=============
+
+Run on client:
+
+    sudo git clone https://git.jim.sh/jim/borg-setup.git /opt/borg
+    sudo /opt/borg/initial-setup.sh
+
+Customize `/opt/borg/backup.yaml` as desired.
+
+
+
+Cheat sheet
+===========
+
+*The copy of this file left on the client will have the variables
+in this section filled in automatically*
+
+### Configuration
+
+    Hostname: ${HOSTNAME}
+    Base directory: ${BORG_DIR}
+    Destination: ${BACKUP_USER}@${BACKUP_HOST}
+    Repository: ${BACKUP_REPO}
+
+### Commands
+
+See when next backup is scheduled:
+
+    systemctl list-timers borg-backup.timer
+
+See progress of most recent backup:
+
+    systemctl status -l -n 99999 borg-backup
+
+Start backup now:
+
+    sudo systemctl start borg-backup
+
+Interrupt backup in progress:
+
+    sudo systemctl stop borg-backup
+
+Show backups and related info:
+
+    sudo ${BORG_DIR}/borg.sh info
+    sudo ${BORG_DIR}/borg.sh list
+
+Run Borg using the read-write SSH key:
+
+    sudo ${BORG_DIR}/borg.sh --rw list
+
+Mount and look at files:
+
+    mkdir mnt
+    sudo ${BORG_DIR}/borg.sh mount :: mnt
+    sudo -s # to explore as root
+    sudo umount mnt
+
+Prune old backups.  Only run if sure local system was never compromised,
+as object deletion could have been queued during append-only operations.
+Requires SSH key password from bitwarden.
+
+    sudo ${BORG_DIR}/prune.sh
+
+
+
+Design
+======
 
 - On server, we have a separate user account "jim-backups".  Password
   for this account is in bitwarden in the "Backups" folder, under `ssh
@@ -33,10 +101,3 @@
   `/opt/borg/backup.yaml` to generate our own list of files, excluding
   anything that's too large by default.  This requires borg 1.2.0b1
   or newer, which is why the setup scripts download a specific version.
-
-# Usage
-
-Run on client:
-
-    sudo git clone https://git.jim.sh/jim/borg-setup.git /opt/borg
-    sudo /opt/borg/initial-setup.sh

backup.py

@@ -1,16 +1,16 @@
-#!/usr/bin/python3
+#!.venv/bin/python
 
 import os
 import sys
 import stat
-from typing import Optional
+from typing import Optional, Tuple
 import humanfriendly # type: ignore
 import wcmatch.glob # type: ignore
 import re
 import dataclasses
 import enum
 
-class MatchResult(Enum):
+class MatchResult(enum.Enum):
     INCLUDE_IF_SIZE_OK = 0
     INCLUDE_ALWAYS = 1
     EXCLUDE_ALWAYS = 2
@@ -20,7 +20,7 @@ class PatternRule:
     re_inc: list[re.Pattern]
     re_exc: list[re.Pattern]
 
-    def match(self, path: str) -> (bool, bool):
+    def match(self, path: str) -> Tuple[bool, bool]:
         if "big" in path:
             print(self, file=sys.stderr)
 

initial-setup.sh

@@ -1,10 +1,21 @@
 #!/bin/bash
 
-BORG_DIR=${BORG_DIR:-/opt/borg}
+# These can be overridden when running this script
 BACKUP_HOST=${BACKUP_HOST:-backup.jim.sh}
 BACKUP_USER=${BACKUP_USER:-jim-backups}
 BACKUP_REPO=${BACKUP_REPO:-borg/$(hostname)}
 
+# Borg binary and hash
+BORG_URL="https://github.com/borgbackup/borg/releases/download/1.2.0b3/borg-linux64"
+BORG_SHA256=8dd6c2769d9bf3ca7a65ebf6781302029fc3b15105aff63d33195c007f897360
+
+# Main dir is where this repo was checked out
+BORG_DIR="$(realpath "$(dirname "$0")")"
+
+# This is named with uppercase so that it doesn't tab-complete for
+# "./b<tab>", which should give us "./borg.sh"
+BORG_BIN="${BORG_DIR}/Borg.bin"
+
 # Use stable host ID in case MAC address changes
 HOSTID="$(hostname -f)@$(python -c 'import uuid;print(uuid.getnode())')"
 
@@ -18,13 +29,13 @@ trap 'error_handler ${BASH_SOURCE} ${LINENO} $?' ERR
 set -o errexit
 set -o errtrace
 
-if [ -e "$BORG_DIR" ]; then
-   echo "Error: BORG_DIR $BORG_DIR already exists; giving up"
-   exit 1
+if [ -e "$BORG_DIR/.setup-complete" ]; then
+    echo "Error: BORG_DIR $BORG_DIR was already set up; giving up."
+    echo "Use \"git clean\" to return it to original state if desired"
+    exit 1
 fi
 
 # Make a temp dir to work in
-mkdir "$BORG_DIR"
 TMP=$(mktemp -d --tmpdir="$BORG_DIR")
 
 # Install some cleanup handlers
@@ -57,18 +68,20 @@ notice() { msg 32 "$@" ; }
 warn() { msg 31 "$@" ; }
 error() { msg 31 "Error:" "$@" ; exit 1 ; }
 
-# Install required packages
-install_dependencies()
+# Create pip environment
+setup_venv()
 {
-    NEED=
-    check() {
-        command -v "$1" >/dev/null || NEED+=" $2"
-    }
-    check borg borgbackup
-    if [ -n "${NEED:+x}" ]; then
-        log "Need to install packages: $NEED"
-        apt install --no-upgrade $NEED
+    ( cd "${BORG_DIR}" && mkdir .venv && pipenv install )
+}
+
+# Install borg
+install_borg()
+{
+    curl -L --progress-bar -o "${BORG_BIN}" "${BORG_URL}"
+    if ! echo "${BORG_SHA256} ${BORG_BIN}" | sha256sum -c ; then
+        error "hash error"
     fi
+    chmod +x "${BORG_BIN}"
 }
 
 # Create wrapper to execute borg
@@ -87,8 +100,16 @@ export BORG_HOST_ID=${HOSTID}
 export BORG_BASE_DIR=${BORG_DIR}
 export BORG_CACHE_DIR=${BORG_DIR}/cache
 export BORG_CONFIG_DIR=${BORG_DIR}/config
-export BORG_RSH="ssh -F $SSH/config -i $SSH/id_ecdsa_appendonly"
-exec borg "\$@"
+if [ "\$1" = "--rw" ] ; then
+    echo "=== Need SSH key passphrase.  Check Bitwarden for:"
+    echo "===   borg $(hostname) / read-write SSH key"
+    export BORG_RSH="ssh -F $SSH/config -o BatchMode=no -i $SSH/id_ecdsa"
+    shift
+else
+    export BORG_RSH="ssh -F $SSH/config -i $SSH/id_ecdsa_appendonly"
+fi
+
+exec "${BORG_BIN}" "\$@"
 EOF
     chmod +x "$BORG"
     if ! "$BORG" -h >/dev/null ; then
@@ -162,7 +183,7 @@ EOF
     # Copy SSH keys to the server's authorized_keys file, removing any
     # existing keys with this HOSTID.
     log "Setting up SSH keys on remote host"
-    cmd="borg serve --restrict-to-repository ~/$BACKUP_REPO"
+    cmd="borg/borg serve --restrict-to-repository ~/$BACKUP_REPO"
 
     keys=".ssh/authorized_keys"
     backup="${keys}.old-$(date +%Y%m%d-%H%M%S)"
@@ -204,68 +225,6 @@ Passphrase: ${PASS_REPOKEY}
 EOF
 }
 
-# Create helper scripts to backup, prune, and mount
-create_scripts()
-{
-    cat > "${BORG_DIR}/backup.sh" <<EOF
-#!/bin/bash
-
-BORG=$BORG_DIR/borg.sh
-set -e
-
-# Explicitly list a bunch of directories to back up, in case they come
-# from different filesystems.  If not, duplicates have no effect.
-DIRS="/"
-for DIR in /usr /var /home /boot /efi ; do
-        if [ -e "\$DIR" ] ; then
-                DIRS="\$DIRS \$DIR"
-        fi
-done
-
-# Allow dirs to be overridden
-BORG_BACKUP_DIRS=\${BORG_BACKUP_DIRS:-\$DIRS}
-
-echo "Backing up: \$BORG_BACKUP_DIRS"
-
-\$BORG create \\
-        --verbose \\
-        --list \\
-        --filter E \\
-        --stats \\
-        --exclude-caches \\
-        --one-file-system \\
-        --checkpoint-interval 900 \\
-        --compression zstd,3 \\
-        ::'{hostname}-{now:%Y%m%d-%H%M%S}' \\
-        \$BORG_BACKUP_DIRS
-
-\$BORG check \\
-        --verbose \\
-        --last 10
-EOF
-
-    cat > "${BORG_DIR}/prune.sh" <<EOF
-#!/bin/bash
-
-BORG=$BORG_DIR/borg.sh
-set -e
-
-echo "=== Need SSH key passphrase.  Check Bitwarden for:"
-echo "===   borg $(hostname) / read-write SSH key"
-\$BORG prune \\
-        --rsh="ssh -F $SSH/config -o BatchMode=no -i $SSH/id_ecdsa" \\
-        --verbose \\
-        --stats \\
-        --keep-within=7d \\
-        --keep-daily=14 \\
-        --keep-weekly=8 \\
-        --keep-monthly=-1
-EOF
-
-    chmod 755 "${BORG_DIR}/backup.sh"
-    chmod 755 "${BORG_DIR}/prune.sh"
-}
-
 configure_systemd()
 {
     TIMER=borg-backup.timer
@@ -295,7 +254,7 @@ Description=Borg backup to ${BACKUP_HOST}
 
 [Service]
 Type=simple
-ExecStart=${BORG_DIR}/backup.sh
+ExecStart=${BORG_DIR}/backup.py
 Nice=10
 IOSchedulingClass=best-effort
 IOSchedulingPriority=6
@@ -324,52 +283,15 @@ EOF
     fi
 }
 
-make_readme()
+update_readme()
 {
-    cat > "${BORG_DIR}/README" <<EOF
-Backup Configuration
---------------------
-
-    Hostname: $(hostname)
- Destination: ${BACKUP_USER}@${BACKUP_HOST}
-  Repository: ${BACKUP_REPO}
-
-Cheat sheet
------------
-
-See when next backup is scheduled:
-
-    systemctl list-timers borg-backup.timer
-
-See progress of most recent backup:
-
-    systemctl status -l -n 99999 borg-backup
-
-Start backup now:
-
-    sudo systemctl start borg-backup
-
-Interrupt backup in progress:
-
-    sudo systemctl stop borg-backup
-
-Show backups and related info:
-
-    sudo ${BORG_DIR}/borg.sh info
-    sudo ${BORG_DIR}/borg.sh list
-
-Mount and look at files:
-
-    mkdir mnt
-    sudo ${BORG_DIR}/borg.sh mount :: mnt
-    sudo -s # to explore as root
-    sudo umount mnt
-
-Prune old backups.  Only run if sure local system was never compromised,
-as object deletion could have been queued during append-only operations.
-Requires SSH key password from bitwarden.
-    sudo ${BORG_DIR}/prune.sh
-EOF
+    sed -i \
+        -e "s!\${HOSTNAME}!$(hostname)!g" \
+        -e "s!\${BORG_DIR}!${BORG_DIR}!g" \
+        -e "s!\${BACKUP_USER}!${BACKUP_USER}!g" \
+        -e "s!\${BACKUP_HOST}!${BACKUP_HOST}!g" \
+        -e "s!\${BACKUP_REPO}!${BACKUP_REPO}!g" \
+        "${BORG_DIR}/README.md"
 }
 
 log "Configuration:"
@@ -377,28 +299,31 @@ log "  Backup server host: ${BACKUP_HOST}"
 log "  Backup server user: ${BACKUP_USER}"
 log "     Repository path: ${BACKUP_REPO}"
 
-install_dependencies
+setup_venv
+install_borg
 create_borg_wrapper
 generate_keys
 configure_ssh
 create_repo
 export_keys
-create_scripts
 configure_systemd
-make_readme
+update_readme
 
 echo
 notice "Add these two passwords to Bitwarden:"
 notice ""
 notice "      Name: borg $(hostname)"
-notice "  Username: repo key"
-notice "  Password: $PASS_REPOKEY"
-notice ""
-notice "      Name: borg $(hostname)"
 notice "  Username: read-write ssh key"
 notice "  Password: $PASS_SSH"
 notice ""
-notice "You should also print out the full repo key: ${BORG_DIR}/key.txt"
+notice "      Name: borg $(hostname)"
+notice "  Username: repo key"
+notice "  Password: $PASS_REPOKEY"
+notice "     Notes: (paste the following key)"
+sed -ne '/BORG/,/^$/{/./p}' "${BORG_DIR}/key.txt"
+notice ""
+notice ""
+
 echo
 
 echo "All done"

prune.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+BORG="$(dirname "$0")/borg.sh --rw"
+set -e
+
+$BORG prune \
+        --verbose \
+        --stats \
+        --keep-within=7d \
+        --keep-daily=14 \
+        --keep-weekly=8 \
+        --keep-monthly=-1