My backup scripts and tools
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

405 lines
9.7 KiB

  1. #!/bin/bash
  2. BORG_DIR=${BORG_DIR:-/opt/borg}
  3. BACKUP_HOST=${BACKUP_HOST:-backup.jim.sh}
  4. BACKUP_USER=${BACKUP_USER:-jim-backups}
  5. BACKUP_REPO=${BACKUP_REPO:-borg/$(hostname)}
  6. # Use stable host ID in case MAC address changes
  7. HOSTID="$(hostname -f)@$(python -c 'import uuid;print(uuid.getnode())')"
  8. function error_handler() {
  9. echo "Error at $1 line $2:"
  10. echo -n '>>> ' ; tail -n +"$2" < "$1" | head -1
  11. echo "... exited with code $3"
  12. exit "$3"
  13. }
  14. trap 'error_handler ${BASH_SOURCE} ${LINENO} $?' ERR
  15. set -o errexit
  16. set -o errtrace
  17. if [ -e "$BORG_DIR" ]; then
  18. echo "Error: BORG_DIR $BORG_DIR already exists; giving up"
  19. exit 1
  20. fi
  21. # Make a temp dir to work in
  22. mkdir "$BORG_DIR"
  23. TMP=$(mktemp -d --tmpdir="$BORG_DIR")
  24. # Install some cleanup handlers
  25. cleanup()
  26. {
  27. set +o errexit
  28. set +o errtrace
  29. trap - ERR
  30. ssh -o ControlPath="$TMP"/ssh-control -O exit x >/dev/null 2>&1
  31. rm -rf -- "$TMP"
  32. }
  33. cleanup_int()
  34. {
  35. echo
  36. cleanup
  37. exit 1
  38. }
  39. trap cleanup 0
  40. trap cleanup_int 1 2 15
  41. msg()
  42. {
  43. color="$1"
  44. shift
  45. echo -ne "\033[1;${color}m===\033[0;${color}m" "$@"
  46. echo -e "\033[0m"
  47. }
  48. log(){ msg 33 "$@" ; }
  49. notice() { msg 32 "$@" ; }
  50. warn() { msg 31 "$@" ; }
  51. error() { msg 31 "Error:" "$@" ; exit 1 ; }
  52. # Install required packages
  53. install_dependencies()
  54. {
  55. NEED=
  56. check() {
  57. command -v "$1" >/dev/null || NEED+=" $2"
  58. }
  59. check borg borgbackup
  60. if [ -n "${NEED:+x}" ]; then
  61. log "Need to install packages: $NEED"
  62. apt install --no-upgrade $NEED
  63. fi
  64. }
  65. # Create wrapper to execute borg
  66. create_borg_wrapper()
  67. {
  68. BORG=${BORG_DIR}/borg.sh
  69. BORG_REPO="ssh://${BACKUP_USER}@${BACKUP_HOST}/./${BACKUP_REPO}"
  70. SSH=$BORG_DIR/ssh
  71. cat >"$BORG" <<EOF
  72. #!/bin/sh
  73. export BORG_REPO=${BORG_REPO}
  74. export BORG_PASSCOMMAND="cat ${BORG_DIR}/passphrase"
  75. export BORG_HOST_ID=${HOSTID}
  76. export BORG_BASE_DIR=${BORG_DIR}
  77. export BORG_CACHE_DIR=${BORG_DIR}/cache
  78. export BORG_CONFIG_DIR=${BORG_DIR}/config
  79. export BORG_RSH="ssh -F $SSH/config -i $SSH/id_ecdsa_appendonly"
  80. exec borg "\$@"
  81. EOF
  82. chmod +x "$BORG"
  83. if ! "$BORG" -h >/dev/null ; then
  84. error "Can't run the new borg wrapper; does borg work?"
  85. fi
  86. }
  87. print_random_key()
  88. {
  89. dd if=/dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 16
  90. }
  91. generate_keys()
  92. {
  93. PASS_SSH=$(print_random_key)
  94. PASS_REPOKEY=$(print_random_key)
  95. echo "$PASS_REPOKEY" > "${BORG_DIR}/passphrase"
  96. chmod 600 "${BORG_DIR}/passphrase"
  97. }
  98. # Run a command on the remote host over an existing SSH tunnel
  99. run_ssh_command()
  100. {
  101. ssh -o ControlPath="$TMP"/ssh-control use-existing-control-tunnel "$@"
  102. }
  103. # Configure SSH key-based login
  104. configure_ssh()
  105. {
  106. mkdir "$SSH"
  107. # Create keys
  108. log "Creating SSH keys"
  109. ssh-keygen -N "" -t ecdsa \
  110. -C "backup-appendonly@$HOSTID" -f "$SSH/id_ecdsa_appendonly"
  111. ssh-keygen -N "$PASS_SSH" -t ecdsa \
  112. -C "backup@$HOSTID" -f "$SSH/id_ecdsa"
  113. # Create config snippets
  114. log "Creating SSH config and wrapper script"
  115. cat >> "$SSH/config" <<EOF
  116. User $BACKUP_USER
  117. ControlPath none
  118. ServerAliveInterval 120
  119. Compression no
  120. UserKnownHostsFile $SSH/known_hosts
  121. ForwardX11 no
  122. ForwardAgent no
  123. BatchMode yes
  124. IdentitiesOnly yes
  125. EOF
  126. # Connect to backup host, using persistent control socket
  127. log "Connecting to server"
  128. log "Please enter password; look in Bitwarden for: ${BACKUP_USER}@${BACKUP_HOST}"
  129. ssh -F "$SSH/config" -o BatchMode=no -o PubkeyAuthentication=no \
  130. -o ControlMaster=yes -o ControlPath="$TMP/ssh-control" \
  131. -o StrictHostKeyChecking=accept-new \
  132. -f "${BACKUP_USER}@${BACKUP_HOST}" sleep 600
  133. if ! run_ssh_command true >/dev/null 2>&1 </dev/null ; then
  134. error "SSH failed"
  135. fi
  136. log "Connected to ${BACKUP_USER}@${BACKUP_HOST}"
  137. # Since we now have an SSH connection, check that the repo doesn't exist
  138. if run_ssh_command "test -e $BACKUP_REPO" ; then
  139. error "$BACKUP_REPO already exists on the server, bailing out"
  140. fi
  141. # Copy SSH keys to the server's authorized_keys file, removing any
  142. # existing keys with this HOSTID.
  143. log "Setting up SSH keys on remote host"
  144. cmd="borg serve --restrict-to-repository ~/$BACKUP_REPO"
  145. keys=".ssh/authorized_keys"
  146. backup="${keys}.old-$(date +%Y%m%d-%H%M%S)"
  147. run_ssh_command "mkdir -p .ssh; chmod 700 .ssh; touch $keys"
  148. run_ssh_command "mv $keys $backup; sed '/@$HOSTID\$/d' < $backup > $keys"
  149. run_ssh_command "if cmp -s $backup $keys; then rm $backup ; fi"
  150. run_ssh_command "cat >> .ssh/authorized_keys" <<EOF
  151. command="$cmd --append-only",restrict $(cat "$SSH/id_ecdsa_appendonly.pub")
  152. command="$cmd",restrict $(cat "$SSH/id_ecdsa.pub")
  153. EOF
  154. # Test that everything worked
  155. log "Testing SSH login with new key"
  156. if ! ssh -F "$SSH/config" -i "$SSH/id_ecdsa_appendonly" -T \
  157. "${BACKUP_USER}@${BACKUP_HOST}" borg --version </dev/null ; then
  158. error "Logging in with a key failed -- is server set up correctly?"
  159. fi
  160. log "Remote connection OK!"
  161. }
  162. # Create the repository on the server
  163. create_repo()
  164. {
  165. log "Creating repo $BACKUP_REPO"
  166. # Create repo
  167. $BORG init --make-parent-dirs --encryption repokey
  168. }
  169. # Export keys as HTML page
  170. export_keys()
  171. {
  172. log "Exporting keys"
  173. $BORG key export --paper '' "${BORG_DIR}/key.txt"
  174. chmod 600 "${BORG_DIR}/key.txt"
  175. cat >>"${BORG_DIR}/key.txt" <<EOF
  176. Repository: ${BORG_REPO}
  177. Passphrase: ${PASS_REPOKEY}
  178. EOF
  179. }
  180. # Create helper scripts to backup, prune, and mount
  181. create_scripts()
  182. {
  183. cat > "${BORG_DIR}/backup.sh" <<EOF
  184. #!/bin/bash
  185. BORG=$BORG_DIR/borg.sh
  186. set -e
  187. # Explicitly list a bunch of directories to back up, in case they come
  188. # from different filesystems. If not, duplicates have no effect.
  189. DIRS="/"
  190. for DIR in /usr /var /home /boot /efi ; do
  191. if [ -e "\$DIR" ] ; then
  192. DIRS="\$DIRS \$DIR"
  193. fi
  194. done
  195. # Allow dirs to be overridden
  196. BORG_BACKUP_DIRS=\${BORG_BACKUP_DIRS:-\$DIRS}
  197. echo "Backing up: \$BORG_BACKUP_DIRS"
  198. \$BORG create \\
  199. --verbose \\
  200. --list \\
  201. --filter E \\
  202. --stats \\
  203. --exclude-caches \\
  204. --one-file-system \\
  205. --checkpoint-interval 900 \\
  206. --compression zstd,3 \\
  207. ::'{hostname}-{now:%Y%m%d-%H%M%S}' \\
  208. \$BORG_BACKUP_DIRS
  209. \$BORG check \\
  210. --verbose \\
  211. --last 10
  212. EOF
  213. cat > "${BORG_DIR}/prune.sh" <<EOF
  214. #!/bin/bash
  215. BORG=$BORG_DIR/borg.sh
  216. set -e
  217. echo "=== Need SSH key passphrase. Check Bitwarden for:"
  218. echo "=== borg $(hostname) / read-write SSH key"
  219. \$BORG prune \\
  220. --rsh="ssh -F $SSH/config -o BatchMode=no -i $SSH/id_ecdsa" \\
  221. --verbose \\
  222. --stats \\
  223. --keep-within=7d \\
  224. --keep-daily=14 \\
  225. --keep-weekly=8 \\
  226. --keep-monthly=-1
  227. EOF
  228. chmod 755 "${BORG_DIR}/backup.sh"
  229. chmod 755 "${BORG_DIR}/prune.sh"
  230. }
  231. configure_systemd()
  232. {
  233. TIMER=borg-backup.timer
  234. SERVICE=borg-backup.service
  235. TIMER_UNIT=${BORG_DIR}/${TIMER}
  236. SERVICE_UNIT=${BORG_DIR}/${SERVICE}
  237. log "Creating systemd files"
  238. cat > "$TIMER_UNIT" <<EOF
  239. [Unit]
  240. Description=Borg backup to ${BACKUP_HOST}
  241. [Timer]
  242. OnCalendar=*-*-* 01:00:00
  243. RandomizedDelaySec=1800
  244. FixedRandomDelay=true
  245. Persistent=true
  246. [Install]
  247. WantedBy=timers.target
  248. EOF
  249. cat >> "$SERVICE_UNIT" <<EOF
  250. [Unit]
  251. Description=Borg backup to ${BACKUP_HOST}
  252. [Service]
  253. Type=simple
  254. ExecStart=${BORG_DIR}/backup.sh
  255. Nice=10
  256. IOSchedulingClass=best-effort
  257. IOSchedulingPriority=6
  258. EOF
  259. log "Setting up systemd"
  260. if (
  261. ln -sfv "${TIMER_UNIT}" /etc/systemd/system &&
  262. ln -sfv "${SERVICE_UNIT}" /etc/systemd/system &&
  263. systemctl --no-ask-password daemon-reload &&
  264. systemctl --no-ask-password enable ${TIMER} &&
  265. systemctl --no-ask-password start ${TIMER}
  266. ); then
  267. log "Backup timer installed:"
  268. systemctl list-timers ${TIMER}
  269. else
  270. warn ""
  271. warn "Systemd setup failed"
  272. warn "Do something like this to configure automatic backups:"
  273. echo " sudo ln -sfv \"${TIMER_UNIT}\" /etc/systemd/system &&"
  274. echo " sudo ln -sfv \"${SERVICE_UNIT}\" /etc/systemd/system &&"
  275. echo " sudo systemctl daemon-reload &&"
  276. echo " sudo systemctl enable ${TIMER} &&"
  277. echo " sudo systemctl start ${TIMER}"
  278. warn ""
  279. fi
  280. }
  281. make_readme()
  282. {
  283. cat > "${BORG_DIR}/README" <<EOF
  284. Backup Configuration
  285. --------------------
  286. Hostname: $(hostname)
  287. Destination: ${BACKUP_USER}@${BACKUP_HOST}
  288. Repository: ${BACKUP_REPO}
  289. Cheat sheet
  290. -----------
  291. See when next backup is scheduled:
  292. systemctl list-timers borg-backup.timer
  293. See progress of most recent backup:
  294. systemctl status -l -n 99999 borg-backup
  295. Start backup now:
  296. sudo systemctl start borg-backup
  297. Interrupt backup in progress:
  298. sudo systemctl stop borg-backup
  299. Show backups and related info:
  300. sudo ${BORG_DIR}/borg.sh info
  301. sudo ${BORG_DIR}/borg.sh list
  302. Mount and look at files:
  303. mkdir mnt
  304. sudo ${BORG_DIR}/borg.sh mount :: mnt
  305. sudo -s # to explore as root
  306. sudo umount mnt
  307. Prune old backups. Only run if sure local system was never compromised,
  308. as object deletion could have been queued during append-only operations.
  309. Requires SSH key password from bitwarden.
  310. sudo ${BORG_DIR}/prune.sh
  311. EOF
  312. }
  313. log "Configuration:"
  314. log " Backup server host: ${BACKUP_HOST}"
  315. log " Backup server user: ${BACKUP_USER}"
  316. log " Repository path: ${BACKUP_REPO}"
  317. install_dependencies
  318. create_borg_wrapper
  319. generate_keys
  320. configure_ssh
  321. create_repo
  322. export_keys
  323. create_scripts
  324. configure_systemd
  325. make_readme
  326. echo
  327. notice "Add these two passwords to Bitwarden:"
  328. notice ""
  329. notice " Name: borg $(hostname)"
  330. notice " Username: repo key"
  331. notice " Password: $PASS_REPOKEY"
  332. notice ""
  333. notice " Name: borg $(hostname)"
  334. notice " Username: read-write ssh key"
  335. notice " Password: $PASS_SSH"
  336. notice ""
  337. notice "You should also print out the full repo key: ${BORG_DIR}/key.txt"
  338. echo
  339. echo "All done"