Fix some exclude/include path issues; misc setup improvements

Makefile

@@ -19,15 +19,16 @@ ctrl: test-backup
 .PHONY: test-backup
 test-backup: .venv
 	.venv/bin/mypy backup.py
-	./backup.py -n
+	./backup.py | tr '\0' '\n' #-n
 
 .PHONY: test-setup
 test-setup:
 	shellcheck -f gcc initial-setup.sh
 	rm -rf /tmp/test-borg
 	mkdir /tmp/test-borg
-	: "normally this would be a git clone, but we want the working tree..."
-	git ls-files -z | tar --null -T - -cf - | tar -C /tmp/test-borg -xvf -
+	git clone . /tmp/test-borg
+	#: "normally this would be a git clone, but we want the working tree..."
+	#git ls-files -z | tar --null -T - -cf - | tar -C /tmp/test-borg -xvf -
 	/tmp/test-borg/initial-setup.sh
 
 .PHONY: clean

README.md

@@ -13,8 +13,8 @@ Customize `/opt/borg/config.yaml` as desired.
 Cheat sheet
 ===========
 
-*The copy of this file left on the client will have the variables
-in this section filled in automatically*
+*After setup, the copy of this file on the client will have the
+variables in this section filled in automatically*
 
 ### Configuration
 

backup.py

@@ -10,6 +10,7 @@ import re
 import sys
 import stat
 import pathlib
+import subprocess
 
 import typing
 
@@ -47,14 +48,14 @@ class Config:
                 for x in raw:
                     if not len(x):
                         continue
-                    if x[0] == b'/':
+                    if x.startswith(b'/'):
                         pats.append(x)
                     else:
                         pats.append(b'**/' + x)
                 return pats
 
             self.exclude = process_match_list('exclude')
-            self.force_include = process_match_list('force_include')
+            self.force_include = process_match_list('force-include')
 
             self.notify_email = config.get('notify-email', None)
 
@@ -78,6 +79,9 @@ class Config:
     def match_re(self, re: tuple[list[typing.Pattern],
                                  list[typing.Pattern]],
                  path: bytes, is_dir: bool):
+        if b'data.ext4.win000' in path:
+            print(path)
+            print(re)
         # If it's a directory, try matching against a trailing slash
         # first.
         if is_dir and self.match_re(re, path + b'/', False):
@@ -166,7 +170,7 @@ class Backup:
                 def format_size(n):
                     return humanfriendly.format_size(
                         n, keep_width=True, binary=True)
-                a = format_size(st.st_size)
+                a = format_size(st.st_blocks * 512)
                 b = format_size(self.config.max_file_size)
                 exclude_reason = ('W', f"file size {a} exceeds limit {b}")
 
@@ -192,7 +196,7 @@ class Backup:
                         with open(path + b'/CACHEDIR.TAG', 'rb') as f:
                             if f.read(len(tag)) == tag:
                                 self.log(
-                                    'I', f"skipping cache dir: {pathstr}")
+                                    'I', f"skipping, cache dir: {pathstr}")
                                 return
                     except:
                         pass
@@ -234,6 +238,20 @@ def main(argv: list[str]):
         with open(os.devnull, "wb") as out:
             backup.run(out)
     else:
+#        subprocess.Popen([args.borg,
+#                          "create",
+#                          "--verbose",
+#                          "--list",
+#                          "--filter", "E",
+#                          "--stats",
+#                          "--exclude-caches"])
+#        --stats \
+#        --exclude-caches \
+#        --one-file-system \
+#        --checkpoint-interval 900 \
+#        --compression zstd,3 \
+#        ::'{hostname}-{now:%Y%m%d-%H%M%S}' \
+#        $BORG_BACKUP_DIRS
         backup.run(sys.stdout.buffer)
 
 if __name__ == "__main__":

config.yaml

@@ -1,4 +1,4 @@
-root: "/tmp/test"
+root: "/"
 one-file-system: true
 exclude-caches: true
 
@@ -24,6 +24,7 @@ exclude: |
 # excluded due to file size or the "exclude" list.
 # Matching rules are the same as above.
 force-include: |
+  .git/objects/pack/*.pack
 
 # Email address for notification at end of backup
 notify-email: jim@jim.sh

initial-setup.sh

@@ -295,6 +295,17 @@ update_readme()
         "${BORG_DIR}/README.md"
 }
 
+git_setup()
+{
+    if ! git checkout -b "setup-$(hostname)" ; then
+        warn "Git setup failed; ignoring"
+        return
+    fi
+
+    log "Committing local changes to git"
+    git status
+}
+
 log "Configuration:"
 log "  Backup server host: ${BACKUP_HOST}"
 log "  Backup server user: ${BACKUP_USER}"
@@ -309,6 +320,7 @@ create_repo
 export_keys
 configure_systemd
 update_readme
+git_setup
 
 echo
 notice "Add these two passwords to Bitwarden:"
@@ -320,11 +332,8 @@ notice ""
 notice "      Name: borg $(hostname)"
 notice "  Username: repo key"
 notice "  Password: $PASS_REPOKEY"
-notice "     Notes: (paste the following key)"
-sed -ne '/BORG/,/^$/{/./p}' "${BORG_DIR}/key.txt"
-notice ""
 notice ""
-notice "Then, test the backup file list with"
+notice "Test the backup file list with"
 notice "    sudo ${BORG_DIR}/backup.py --dry-run"
 notice "and make any necessary adjustments to:"
 notice "    ${BORG_DIR}/config.yaml"