backup: show detailed progress from borg

backup.py

@@ -8,6 +8,7 @@
 import os
 import re
 import sys
+import json
 import stat
 import time
 import select
@@ -24,6 +25,9 @@ import humanfriendly # type: ignore
 def pstr(path: bytes) -> str:
     return path.decode(errors='backslashreplace')
 
+def format_size(n: int) -> str:
+    return humanfriendly.format_size(n, keep_width=True, binary=True)
+
 class Config:
     roots: list[bytes]
     max_file_size: typing.Optional[int]
@@ -174,9 +178,6 @@ class Backup:
                   and self.config.max_file_size
                   and size > self.config.max_file_size):
                 # Too big
-                def format_size(n):
-                    return humanfriendly.format_size(
-                        n, keep_width=True, binary=True)
                 a = format_size(size)
                 b = format_size(self.config.max_file_size)
                 exclude_reason = ('W', f"file size {a} exceeds limit {b}")
@@ -287,6 +288,8 @@ def main(argv: list[str]):
         borg = subprocess.Popen([borg,
                                  "create",
                                  "--verbose",
+                                 "--progress",
+                                 "--log-json",
                                  "--list",
                                  "--filter", "E",
                                  "--stats",
@@ -303,17 +306,38 @@ def main(argv: list[str]):
 
         # Use a thread to capture output
         def reader_thread(fh):
-            os.set_blocking(fh.fileno(), False)
-            while True:
-                ready = select.select([fh.fileno()], [], [])
-                if not len(ready[0]):
-                    break
-                data = fh.read(8192)
-                if not len(data):
-                    break
-                sys.stdout.buffer.write(data)
+            last_progress = 0
+            for line in fh:
+                try:
+                    data = json.loads(line)
+                    if ((data['type'] == 'log_message' or
+                         data['type'] == 'progress_message')
+                        and 'message' in data):
+                        line = (data['message'] + '\n').encode()
+                    elif data['type'] == 'archive_progress':
+                        now = time.time()
+                        if now - last_progress > 10:
+                            last_progress = now
+                            def size(short: str, full: str) -> str:
+                                return f" {short}={format_size(data[full])}"
+                            line = (f"progress:" +
+                                    f" files={data['nfiles']}" +
+                                    size('orig', 'original_size') +
+                                    size('comp', 'compressed_size') +
+                                    size('dedup', 'deduplicated_size') +
+                                    f" path={data['path']}" +
+                                    "\n").encode()
+                        else:
+                            continue
+                    else:
+                        # ignore unknown progress line
+                        continue
+                except Exception as e:
+                    # on error, print raw line
+                    pass
+                sys.stdout.buffer.write(line)
                 sys.stdout.flush()
-                captured_output.append(data)
+                captured_output.append(line)
             fh.close()
         reader = threading.Thread(target=reader_thread, args=(borg.stdout,))
         reader.daemon = True