backup: replace simple max size with rule-based system

Now individual files or patterns can have their own maximum sizes.

backup.py

@@ -36,18 +36,18 @@ MatchPatterns = typing.Tuple[typing.List[re.Pattern], typing.List[re.Pattern]]
 
 class Config:
     roots: typing.List[bytes]
-    max_file_size: typing.Optional[int]
     one_file_system: bool
     exclude_caches: bool
     exclude: MatchPatterns
     unexclude: MatchPatterns
+    max_size_rules: typing.List[typing.Tuple[int, MatchPatterns]]
     notify_email: typing.Optional[str]
 
     def __init__(self, configfile: str):
 
         # Helper to process lists of patterns into regexes
-        def process_match_list(config_name):
-                raw = config.get(config_name, '').encode().split(b'\n')
+        def process_match_list(config_entry):
+                raw = config_entry.encode().split(b'\n')
                 pats = []
                 # Prepend '**/' to any relative patterns
                 for x in raw:
@@ -74,12 +74,6 @@ class Config:
             self.one_file_system = config.get('one-file-system', False)
             self.exclude_caches = config.get('exclude-caches', False)
 
-            if 'max-file-size' in config:
-                self.max_file_size = humanfriendly.parse_size(
-                    config['max-file-size'])
-            else:
-                self.max_file_size = None
-
             raw = config.get('roots', '').encode().split(b'\n')
             self.roots = []
             for x in raw:
@@ -88,8 +82,15 @@ class Config:
                 self.roots.append(x)
             self.roots.sort(key=len)
 
-            self.exclude = process_match_list('exclude')
-            self.unexclude = process_match_list('unexclude')
+            self.exclude = process_match_list(config.get('exclude', ''))
+            self.unexclude = process_match_list(config.get('unexclude', ''))
+
+            self.max_size_rules = []
+            rules = { humanfriendly.parse_size(k): v
+                      for k, v in config.get('max-size-rules', {}).items() }
+            for size in reversed(sorted(rules)):
+                self.max_size_rules.append(
+                    (size, process_match_list(rules[size])))
 
             self.notify_email = config.get('notify-email', None)
 
@@ -174,12 +175,20 @@ class Backup:
                 exclude_reason = ('I', "skipping, on different filesystem")
 
             elif (is_reg
-                  and self.config.max_file_size
-                  and size > self.config.max_file_size):
-                # Too big
-                a = format_size(size)
-                b = format_size(self.config.max_file_size)
-                exclude_reason = ('W', f"file size {a} exceeds limit {b}")
+                  and len(self.config.max_size_rules)
+                  and size > self.config.max_size_rules[-1][0]):
+                # Check file sizes against our list.
+                # Only need to check if the size is bigger than the smallest
+                # entry on the list; then, we need to check it against all rules
+                # to see which one applies.
+                for (max_size, patterns) in self.config.max_size_rules:
+                    if self.config.match_re(patterns, decorated_path):
+                        if size > max_size:
+                            a = format_size(size)
+                            b = format_size(max_size)
+                            exclude_reason = (
+                                'W', f"file size {a} exceeds limit {b}")
+                        break
 
             # If we have a reason to exclude it, stop now unless it's
             # force-included

config.yaml

@@ -10,12 +10,6 @@ roots: |
 one-file-system: true
 exclude-caches: true
 
-# Files larger than this are excluded.  If a large file isn't
-# explicitly mentioned in "excludes" below, it also generates a
-# warning.  Note that this counts used blocks, so files with large
-# holes will still be considered small (since they'll compress easily)
-max-file-size: 500MiB
-
 # Files/dirs to exclude from backup.
 # Relative paths are treated as if starting with **/
 # Paths ending in / will only match directories.
@@ -27,9 +21,20 @@ exclude: |
   Steam/ubuntu*/
   .cache/
 
+# Rules to exclude files based on file size.
+# This is a dict of sizes, each with a list of rules.
+# For a given path, the largest size with a matching rule applies.
+# Matching follows the same behavior as the "exclude" list.
+# Size is calculated as used blocks (think "du", not "du --apparent-size").
+max-size-rules:
+  500 MiB: |
+    *
+#  1.0 GiB: |
+#    *.mp4
+
 # Files that are always included, even if they would have been
 # excluded due to file size or the "exclude" list.
-# Matching rules are the same as above.
+# Matching follows the same behavior as the "exclude" list.
 unexclude: |
   .git/objects/pack/*.pack