My backup scripts and tools
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

399 lines
11 KiB

  1. #!/bin/bash
  2. # These can be overridden when running this script
  3. HOSTNAME=${HOSTNAME:-$(hostname)}
  4. BACKUP_HOST=${BACKUP_HOST:-backup.jim.sh}
  5. BACKUP_USER=${BACKUP_USER:-jim-backups}
  6. BACKUP_REPO=${BACKUP_REPO:-borg/${HOSTNAME}}
  7. # Main dir is where this repo was checked out
  8. BORG_DIR="$(realpath "$(dirname "$0")")"
  9. cd "${BORG_DIR}"
  10. BORG_BIN="${BORG_DIR}/bin/borg.$(uname -m)"
  11. # Use stable host ID in case MAC address changes.
  12. # Note that this host ID is only used to manage locks, so it's
  13. # not crucial that it remains stable.
  14. HOSTID="${HOSTNAME}@$(python3 -c 'import uuid;print(uuid.getnode())')"
  15. function error_handler() {
  16. echo "Error at $1 line $2:"
  17. echo -n '>>> ' ; tail -n +"$2" < "$1" | head -1
  18. echo "... exited with code $3"
  19. exit "$3"
  20. }
  21. trap 'error_handler ${BASH_SOURCE} ${LINENO} $?' ERR
  22. set -o errexit
  23. set -o errtrace
  24. # Create pip environment
  25. setup_venv()
  26. {
  27. if ! which pipenv >/dev/null 2>&1 ; then
  28. echo "pipenv not found, try: sudo apt install pipenv"
  29. exit 1
  30. fi
  31. mkdir -p .venv
  32. pipenv install
  33. }
  34. # Create shell script with environment variables
  35. create_borg_vars()
  36. {
  37. VARS=${BORG_DIR}/vars.sh
  38. # These variables are used elsewhere in this script
  39. BORG_REPO="ssh://${BACKUP_USER}@${BACKUP_HOST}/./${BACKUP_REPO}"
  40. BORG=${BORG_DIR}/borg.sh
  41. SSH=$BORG_DIR/ssh
  42. cat >"$VARS" <<EOF
  43. export BACKUP_USER=${BACKUP_USER}
  44. export BACKUP_HOST=${BACKUP_HOST}
  45. export BACKUP_REPO=${BACKUP_REPO}
  46. export HOSTNAME=${HOSTNAME}
  47. export BORG_REPO=${BORG_REPO}
  48. export BORG_HOST_ID=${HOSTID}
  49. export BORG_PASSCOMMAND="cat ${BORG_DIR}/passphrase"
  50. export BORG_DIR=${BORG_DIR}
  51. export SSH=${SSH}
  52. export BORG=${BORG}
  53. export BORG_BIN=${BORG_BIN}
  54. EOF
  55. if ! "$BORG" -h >/dev/null ; then
  56. error "Can't run the borg wrapper; does borg work?"
  57. fi
  58. }
  59. # Update paths in README and backup.py
  60. update_paths()
  61. {
  62. sed -i \
  63. -e "s!\${HOSTNAME}!${HOSTNAME}!g" \
  64. -e "s!\${BORG_DIR}!${BORG_DIR}!g" \
  65. -e "s!\${BORG_BIN}!${BORG_BIN}!g" \
  66. -e "s!\${BACKUP_USER}!${BACKUP_USER}!g" \
  67. -e "s!\${BACKUP_HOST}!${BACKUP_HOST}!g" \
  68. -e "s!\${BACKUP_REPO}!${BACKUP_REPO}!g" \
  69. README.md
  70. sed -i\
  71. -e "1c#!${BORG_DIR}/.venv/bin/python" \
  72. backup.py
  73. }
  74. # See if we're just supposed to update an existing install, or recovering
  75. RECOVER=0
  76. if [ "$1" == "--recover" ] ; then
  77. if [ -e "vars.sh" ]; then
  78. echo "It looks like this borg was already set up, can only recover from fresh start"
  79. exit 1
  80. fi
  81. RECOVER=1
  82. elif [ "$1" == "--update-paths" ] || [ "$1" == "--update" ] ; then
  83. if [ -e "vars.sh" ]; then
  84. echo "Updating paths and variables"
  85. update_paths
  86. setup_venv
  87. create_borg_vars
  88. exit 0
  89. else
  90. echo "Can't update, not set up yet"
  91. exit 1
  92. fi
  93. elif [ -e "vars.sh" ]; then
  94. echo "Error: BORG_DIR $BORG_DIR already looks set up; giving up."
  95. echo "Use \"git clean\" to return it to original state if desired"
  96. exit 1
  97. fi
  98. # Make a temp dir to work in
  99. TMP=$(mktemp -d)
  100. # Install some cleanup handlers
  101. cleanup()
  102. {
  103. set +o errexit
  104. set +o errtrace
  105. trap - ERR
  106. ssh -o ControlPath="$TMP"/ssh-control -O exit x >/dev/null 2>&1
  107. rm -rf -- "$TMP"
  108. }
  109. cleanup_int()
  110. {
  111. echo
  112. cleanup
  113. exit 1
  114. }
  115. trap cleanup 0
  116. trap cleanup_int 1 2 15
  117. msg()
  118. {
  119. color="$1"
  120. shift
  121. echo -ne "\033[1;${color}m===\033[0;${color}m" "$@"
  122. echo -e "\033[0m"
  123. }
  124. log(){ msg 33 "$@" ; }
  125. notice() { msg 32 "$@" ; }
  126. warn() { msg 31 "$@" ; }
  127. error() { msg 31 "Error:" "$@" ; exit 1 ; }
  128. print_random_key()
  129. {
  130. dd if=/dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 16
  131. }
  132. generate_keys()
  133. {
  134. if [ $RECOVER -eq 1 ] ; then
  135. echo "Recovering configuration in order to use an existing backup"
  136. read -s -p "Repo key for \"borg ${HOSTNAME}\": " PASS_REPOKEY
  137. echo
  138. read -s -p "Again: " PASS_REPOKEY2
  139. echo
  140. if [ -z "$PASS_REPOKEY" ] || [ $PASS_REPOKEY != $PASS_REPOKEY2 ] ; then
  141. echo "Bad repo key"
  142. exit 1
  143. fi
  144. else
  145. PASS_REPOKEY=$(print_random_key)
  146. fi
  147. echo "$PASS_REPOKEY" > passphrase
  148. chmod 600 passphrase
  149. }
  150. # Run a command on the remote host over an existing SSH tunnel
  151. run_ssh_command()
  152. {
  153. ssh -o ControlPath="$TMP"/ssh-control use-existing-control-tunnel "$@"
  154. }
  155. # Configure SSH key-based login
  156. configure_ssh()
  157. {
  158. mkdir "$SSH"
  159. # Create keys
  160. log "Creating SSH keys"
  161. ssh-keygen -N "" -t ecdsa \
  162. -C "backup-appendonly@$HOSTID" -f "$SSH/id_ecdsa_appendonly"
  163. ssh-keygen -N "" -t ecdsa \
  164. -C "backup-notify@$HOSTID" -f "$SSH/id_ecdsa_notify"
  165. # Create config snippets
  166. log "Creating SSH config and wrapper script"
  167. cat >> "$SSH/config" <<EOF
  168. User $BACKUP_USER
  169. ControlPath none
  170. ServerAliveInterval 120
  171. Compression no
  172. UserKnownHostsFile $SSH/known_hosts
  173. ForwardX11 no
  174. ForwardAgent no
  175. BatchMode yes
  176. IdentitiesOnly yes
  177. EOF
  178. # Connect to backup host, using persistent control socket
  179. log "Connecting to server"
  180. log "Please enter password; look in Bitwarden for: ssh ${BACKUP_HOST} / ${BACKUP_USER}"
  181. ssh -F "$SSH/config" -o BatchMode=no -o PubkeyAuthentication=no \
  182. -o ControlMaster=yes -o ControlPath="$TMP/ssh-control" \
  183. -o StrictHostKeyChecking=accept-new \
  184. -f "${BACKUP_USER}@${BACKUP_HOST}" sleep 600
  185. if ! run_ssh_command true >/dev/null 2>&1 </dev/null ; then
  186. error "SSH failed"
  187. fi
  188. log "Connected to ${BACKUP_USER}@${BACKUP_HOST}"
  189. # Since we now have an SSH connection, check repo existence
  190. if [ $RECOVER -eq 0 ] && run_ssh_command "test -e $BACKUP_REPO"; then
  191. error "$BACKUP_REPO already exists on the server, bailing out"
  192. elif [ $RECOVER -ne 0 ] && ! run_ssh_command "test -e $BACKUP_REPO"; then
  193. error "$BACKUP_REPO does NOT exist on the server, can't recover backup config"
  194. fi
  195. # Copy SSH keys to the server's authorized_keys file, removing any
  196. # existing keys with this HOSTID.
  197. log "Setting up SSH keys on remote host"
  198. REMOTE_BORG="borg/borg"
  199. cmd="$REMOTE_BORG serve --restrict-to-repository ~/$BACKUP_REPO"
  200. keys=".ssh/authorized_keys"
  201. backup="${keys}.old-$(date +%Y%m%d-%H%M%S)"
  202. run_ssh_command "mkdir -p .ssh; chmod 700 .ssh; touch $keys"
  203. run_ssh_command "mv $keys $backup; sed '/@$HOSTID\$/d' < $backup > $keys"
  204. run_ssh_command "if cmp -s $backup $keys; then rm $backup ; fi"
  205. run_ssh_command "cat >> .ssh/authorized_keys" <<EOF
  206. command="$cmd --append-only",restrict $(cat "$SSH/id_ecdsa_appendonly.pub")
  207. command="borg/notify.sh",restrict $(cat "$SSH/id_ecdsa_notify.pub")
  208. EOF
  209. # Test that everything worked
  210. log "Testing SSH login with new key"
  211. if ! ssh -F "$SSH/config" -i "$SSH/id_ecdsa_appendonly" -T \
  212. "${BACKUP_USER}@${BACKUP_HOST}" "$REMOTE_BORG" --version </dev/null ; then
  213. error "Logging in with a key failed -- is server set up correctly?"
  214. fi
  215. log "Remote connection OK!"
  216. }
  217. # Create the repository on the server
  218. create_repo()
  219. {
  220. log "Creating repo $BACKUP_REPO"
  221. # Create repo
  222. $BORG init --make-parent-dirs --encryption repokey
  223. }
  224. # Export keys as HTML page
  225. export_keys()
  226. {
  227. log "Exporting keys"
  228. $BORG key export --paper '' key.txt
  229. chmod 600 key.txt
  230. cat >>key.txt <<EOF
  231. Repository: ${BORG_REPO}
  232. Passphrase: ${PASS_REPOKEY}
  233. EOF
  234. }
  235. configure_systemd()
  236. {
  237. TIMER=borg-backup.timer
  238. SERVICE=borg-backup.service
  239. TIMER_UNIT=${BORG_DIR}/${TIMER}
  240. SERVICE_UNIT=${BORG_DIR}/${SERVICE}
  241. log "Creating systemd files"
  242. # Choose a time between 1am and 6am based on this hostname
  243. HASH=$(echo hash of "$HOSTNAME" | sha1sum)
  244. HOUR=$((0x${HASH:0:8} % 5 + 1))
  245. MINUTE=$((0x${HASH:8:8} % 6 * 10))
  246. TIME=$(printf %02d:%02d:00 $HOUR $MINUTE)
  247. log "Backup time is $TIME"
  248. cat > "$TIMER_UNIT" <<EOF
  249. [Unit]
  250. Description=Borg backup to ${BACKUP_HOST}
  251. [Timer]
  252. OnCalendar=*-*-* $TIME
  253. Persistent=true
  254. [Install]
  255. WantedBy=timers.target
  256. EOF
  257. cat >> "$SERVICE_UNIT" <<EOF
  258. [Unit]
  259. Description=Borg backup to ${BACKUP_HOST}
  260. [Service]
  261. Type=simple
  262. ExecStart=${BORG_DIR}/backup.py
  263. Nice=10
  264. IOSchedulingClass=best-effort
  265. IOSchedulingPriority=6
  266. Restart=on-failure
  267. RestartSec=600
  268. EOF
  269. if [ $RECOVER -eq 1 ] ; then
  270. log "Partially setting up systemd"
  271. ln -sfv "${TIMER_UNIT}" /etc/systemd/system
  272. ln -sfv "${SERVICE_UNIT}" /etc/systemd/system
  273. systemctl --no-ask-password daemon-reload
  274. systemctl --no-ask-password stop ${TIMER}
  275. systemctl --no-ask-password disable ${TIMER}
  276. warn "Since we're recovering, systemd automatic backups aren't enabled"
  277. warn "Do something like this to configure automatic backups:"
  278. echo " sudo systemctl enable ${TIMER} &&"
  279. echo " sudo systemctl start ${TIMER}"
  280. warn ""
  281. else
  282. log "Setting up systemd"
  283. if (
  284. ln -sfv "${TIMER_UNIT}" /etc/systemd/system &&
  285. ln -sfv "${SERVICE_UNIT}" /etc/systemd/system &&
  286. systemctl --no-ask-password daemon-reload &&
  287. systemctl --no-ask-password enable ${TIMER} &&
  288. systemctl --no-ask-password start ${TIMER}
  289. ); then
  290. log "Backup timer installed:"
  291. systemctl list-timers --no-pager ${TIMER}
  292. else
  293. warn ""
  294. warn "Systemd setup failed"
  295. warn "Do something like this to configure automatic backups:"
  296. echo " sudo ln -sfv \"${TIMER_UNIT}\" /etc/systemd/system &&"
  297. echo " sudo ln -sfv \"${SERVICE_UNIT}\" /etc/systemd/system &&"
  298. echo " sudo systemctl daemon-reload &&"
  299. echo " sudo systemctl enable ${TIMER} &&"
  300. echo " sudo systemctl start ${TIMER}"
  301. warn ""
  302. fi
  303. fi
  304. }
  305. git_setup()
  306. {
  307. if ! git checkout -b "setup-${HOSTNAME}" ; then
  308. warn "Git setup failed; ignoring"
  309. return
  310. fi
  311. log "Committing local changes to git"
  312. git add README.md borg-backup.service borg-backup.timer vars.sh
  313. git commit -a -m "autocommit after initial setup on ${HOSTNAME}"
  314. }
  315. log "Configuration:"
  316. log " Backup server host: ${BACKUP_HOST}"
  317. log " Backup server user: ${BACKUP_USER}"
  318. log " Repository path: ${BACKUP_REPO}"
  319. setup_venv
  320. create_borg_vars
  321. generate_keys
  322. configure_ssh
  323. [ $RECOVER -eq 0 ] && create_repo
  324. export_keys
  325. configure_systemd
  326. update_paths
  327. git_setup
  328. echo
  329. if [ $RECOVER -eq 1 ] ; then
  330. notice "You should be set up with borg pointing to the existing repo now."
  331. notice "Use commands like these to look at the backup:"
  332. notice " sudo /opt/borg/borg.sh info"
  333. notice " sudo /opt/borg/borg.sh list"
  334. notice "You'll want to now restore files like ${BORG_DIR}/config.yaml before enabling systemd timers"
  335. else
  336. notice "Add this password to Bitwarden:"
  337. notice ""
  338. notice " Name: borg ${HOSTNAME}"
  339. notice " Username: repo key"
  340. notice " Password: $PASS_REPOKEY"
  341. notice ""
  342. notice "Test the backup file list with"
  343. notice " sudo ${BORG_DIR}/backup.py --dry-run"
  344. notice "and make any necessary adjustments to:"
  345. notice " ${BORG_DIR}/config.yaml"
  346. fi
  347. echo
  348. echo "All done"