diff --git a/.gitignore b/.gitignore index 2927db0..285527d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -README.html +*.html + diff --git a/Makefile b/Makefile index 214b55b..6f3d2ff 100644 --- a/Makefile +++ b/Makefile @@ -10,21 +10,26 @@ all: @echo .PHONY: ctrl -ctrl: test +ctrl: test-setup -.PHONY: test-lister -test-lister: .venv - venv/bin/mypy lister.py - venv/bin/python lister.py --max-size 1GiB --one-file-system /tmp | grep -a 'bigf' +.venv: + mkdir .venv + pipenv install --dev -.PHONY: check -check: - shellcheck -f gcc initial-setup.sh +.PHONY: test-backup +test-backup: .venv + .venv/bin/mypy backup.py + ./backup.py --max-size 1GiB --one-file-system /tmp | grep -a 'bigf' -.PHONY: test -test: check +.PHONY: test-setup +test-setup: + shellcheck -f gcc initial-setup.sh rm -rf /tmp/test-borg mkdir /tmp/test-borg : "normally this would be a git clone, but we want the working tree..." git ls-files -z | tar --null -T - -cf - | tar -C /tmp/test-borg -xvf - /tmp/test-borg/initial-setup.sh + +.PHONY: clean +clean: + rm -f README.html diff --git a/Pipfile b/Pipfile index cf118cf..2edfce6 100644 --- a/Pipfile +++ b/Pipfile @@ -5,9 +5,9 @@ name = "pypi" [packages] humanfriendly = "*" -mypy = "*" [dev-packages] +mypy = "*" [requires] -python_version = "3.9" +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index b5b2a44..4a0b184 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "775048a9d9eea3ab29a1e53636271f45f9fe40ec250225818155d3eced6034e7" + "sha256": "4f504c785e3ed5b203a82a5f40516507f80a01b8d1d0ad5a905f139cafc41a51" }, "pipfile-spec": 6, "requires": { - "python_version": "3.9" + "python_version": "3" }, "sources": [ { @@ -23,7 +23,9 @@ ], "index": "pypi", "version": "==10.0" - }, + } + }, + "develop": { "mypy": { "hashes": [ "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9", @@ -75,6 +77,5 @@ ], "version": "==3.10.0.2" } - }, - "develop": {} + } } diff --git a/README.md b/README.md index 22c0ea9..d19b608 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,72 @@ -# Design +Initial setup +============= + +Run on client: + + sudo git clone https://git.jim.sh/jim/borg-setup.git /opt/borg + sudo /opt/borg/initial-setup.sh + +Customize `/opt/borg/backup.yaml` as desired. + + + +Cheat sheet +=========== + +*The copy of this file left on the client will have the variables +in this section filled in automatically* + +### Configuration + + Hostname: ${HOSTNAME} + Base directory: ${BORG_DIR} + Destination: ${BACKUP_USER}@${BACKUP_HOST} + Repository: ${BACKUP_REPO} + +### Commands + +See when next backup is scheduled: + + systemctl list-timers borg-backup.timer + +See progress of most recent backup: + + systemctl status -l -n 99999 borg-backup + +Start backup now: + + sudo systemctl start borg-backup + +Interrupt backup in progress: + + sudo systemctl stop borg-backup + +Show backups and related info: + + sudo ${BORG_DIR}/borg.sh info + sudo ${BORG_DIR}/borg.sh list + +Run Borg using the read-write SSH key: + + sudo ${BORG_DIR}/borg.sh --rw list + +Mount and look at files: + + mkdir mnt + sudo ${BORG_DIR}/borg.sh mount :: mnt + sudo -s # to explore as root + sudo umount mnt + +Prune old backups. Only run if sure local system was never compromised, +as object deletion could have been queued during append-only operations. +Requires SSH key password from bitwarden. + + sudo ${BORG_DIR}/prune.sh + + + +Design +====== - On server, we have a separate user account "jim-backups". Password for this account is in bitwarden in the "Backups" folder, under `ssh @@ -33,10 +101,3 @@ `/opt/borg/backup.yaml` to generate our own list of files, excluding anything that's too large by default. This requires borg 1.2.0b1 or newer, which is why the setup scripts download a specific version. - -# Usage - -Run on client: - - sudo git clone https://git.jim.sh/jim/borg-setup.git /opt/borg - sudo /opt/borg/initial-setup.sh diff --git a/lister.py b/backup.py similarity index 97% rename from lister.py rename to backup.py index f0c9af5..c1ae36f 100755 --- a/lister.py +++ b/backup.py @@ -1,16 +1,16 @@ -#!/usr/bin/python3 +#!.venv/bin/python import os import sys import stat -from typing import Optional +from typing import Optional, Tuple import humanfriendly # type: ignore import wcmatch.glob # type: ignore import re import dataclasses import enum -class MatchResult(Enum): +class MatchResult(enum.Enum): INCLUDE_IF_SIZE_OK = 0 INCLUDE_ALWAYS = 1 EXCLUDE_ALWAYS = 2 @@ -20,7 +20,7 @@ class PatternRule: re_inc: list[re.Pattern] re_exc: list[re.Pattern] - def match(self, path: str) -> (bool, bool): + def match(self, path: str) -> Tuple[bool, bool]: if "big" in path: print(self, file=sys.stderr) diff --git a/initial-setup.sh b/initial-setup.sh old mode 100644 new mode 100755 index 3830c98..64e0622 --- a/initial-setup.sh +++ b/initial-setup.sh @@ -1,10 +1,21 @@ #!/bin/bash -BORG_DIR=${BORG_DIR:-/opt/borg} +# These can be overridden when running this script BACKUP_HOST=${BACKUP_HOST:-backup.jim.sh} BACKUP_USER=${BACKUP_USER:-jim-backups} BACKUP_REPO=${BACKUP_REPO:-borg/$(hostname)} +# Borg binary and hash +BORG_URL="https://github.com/borgbackup/borg/releases/download/1.2.0b3/borg-linux64" +BORG_SHA256=8dd6c2769d9bf3ca7a65ebf6781302029fc3b15105aff63d33195c007f897360 + +# Main dir is where this repo was checked out +BORG_DIR="$(realpath "$(dirname "$0")")" + +# This is named with uppercase so that it doesn't tab-complete for +# "./b", which should give us "./borg.sh" +BORG_BIN="${BORG_DIR}/Borg.bin" + # Use stable host ID in case MAC address changes HOSTID="$(hostname -f)@$(python -c 'import uuid;print(uuid.getnode())')" @@ -18,13 +29,13 @@ trap 'error_handler ${BASH_SOURCE} ${LINENO} $?' ERR set -o errexit set -o errtrace -if [ -e "$BORG_DIR" ]; then - echo "Error: BORG_DIR $BORG_DIR already exists; giving up" - exit 1 +if [ -e "$BORG_DIR/.setup-complete" ]; then + echo "Error: BORG_DIR $BORG_DIR was already set up; giving up." + echo "Use \"git clean\" to return it to original state if desired" + exit 1 fi # Make a temp dir to work in -mkdir "$BORG_DIR" TMP=$(mktemp -d --tmpdir="$BORG_DIR") # Install some cleanup handlers @@ -57,18 +68,20 @@ notice() { msg 32 "$@" ; } warn() { msg 31 "$@" ; } error() { msg 31 "Error:" "$@" ; exit 1 ; } -# Install required packages -install_dependencies() +# Create pip environment +setup_venv() +{ + ( cd "${BORG_DIR}" && mkdir .venv && pipenv install ) +} + +# Install borg +install_borg() { - NEED= - check() { - command -v "$1" >/dev/null || NEED+=" $2" - } - check borg borgbackup - if [ -n "${NEED:+x}" ]; then - log "Need to install packages: $NEED" - apt install --no-upgrade $NEED + curl -L --progress-bar -o "${BORG_BIN}" "${BORG_URL}" + if ! echo "${BORG_SHA256} ${BORG_BIN}" | sha256sum -c ; then + error "hash error" fi + chmod +x "${BORG_BIN}" } # Create wrapper to execute borg @@ -87,8 +100,16 @@ export BORG_HOST_ID=${HOSTID} export BORG_BASE_DIR=${BORG_DIR} export BORG_CACHE_DIR=${BORG_DIR}/cache export BORG_CONFIG_DIR=${BORG_DIR}/config -export BORG_RSH="ssh -F $SSH/config -i $SSH/id_ecdsa_appendonly" -exec borg "\$@" +if [ "\$1" = "--rw" ] ; then + echo "=== Need SSH key passphrase. Check Bitwarden for:" + echo "=== borg $(hostname) / read-write SSH key" + export BORG_RSH="ssh -F $SSH/config -o BatchMode=no -i $SSH/id_ecdsa" + shift +else + export BORG_RSH="ssh -F $SSH/config -i $SSH/id_ecdsa_appendonly" +fi + +exec "${BORG_BIN}" "\$@" EOF chmod +x "$BORG" if ! "$BORG" -h >/dev/null ; then @@ -162,7 +183,7 @@ EOF # Copy SSH keys to the server's authorized_keys file, removing any # existing keys with this HOSTID. log "Setting up SSH keys on remote host" - cmd="borg serve --restrict-to-repository ~/$BACKUP_REPO" + cmd="borg/borg serve --restrict-to-repository ~/$BACKUP_REPO" keys=".ssh/authorized_keys" backup="${keys}.old-$(date +%Y%m%d-%H%M%S)" @@ -204,68 +225,6 @@ Passphrase: ${PASS_REPOKEY} EOF } -# Create helper scripts to backup, prune, and mount -create_scripts() -{ - cat > "${BORG_DIR}/backup.sh" < "${BORG_DIR}/prune.sh" < "${BORG_DIR}/README" <