Browse Source

backup: call prune after backup; add run_borg helper

Automatically prunes after backup, although this doesn't actually
free up space (because we're in append-only mode).
master
Jim Paris 7 months ago
parent
commit
e85e08cace
1 changed files with 107 additions and 80 deletions
  1. +107
    -80
      backup.py

+ 107
- 80
backup.py View File

@@ -114,6 +114,9 @@ class Backup:
# Saved log messages
self.logs: typing.List[typing.Tuple[str, str]] = []

# All captured borg output
self.captured_output: typing.List[bytes] = []

def out(self, path: bytes):
self.outfile.write(path + (b'\n' if self.dry_run else b'\0'))

@@ -235,77 +238,13 @@ class Backup:
self.log('E', f"can't read {pstr(path)}: {str(e)}")
return

def main(argv: typing.List[str]):
import argparse

def humansize(string):
return humanfriendly.parse_size(string)

# Parse args
parser = argparse.ArgumentParser(
prog=argv[0],
description="Back up the local system using borg",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)

base = pathlib.Path(__file__).parent
parser.add_argument('-c', '--config',
help="Config file", default=str(base / "config.yaml"))
parser.add_argument('-v', '--vars',
help="Variables file", default=str(base / "vars.sh"))
parser.add_argument('-n', '--dry-run', action="store_true",
help="Just print log output, don't run borg")
parser.add_argument('-d', '--debug', action="store_true",
help="Print filenames for --dry-run")

args = parser.parse_args()
config = Config(args.config)
backup = Backup(config, args.dry_run)

# Parse variables from vars.sh
hostname = os.uname().nodename
borg_sh = str(base / "borg.sh")
notify_sh = str(base / "notify.sh")
try:
with open(args.vars) as f:
for line in f:
m = re.match(r"\s*export\s*([A-Z_]+)=(.*)", line)
if not m:
continue
var = m.group(1)
value = m.group(2)
if var == "HOSTNAME":
hostname = value
if var == "BORG":
borg_sh = value
if var == "BORG_DIR":
notify_sh = str(pathlib.Path(value) / "notify.sh")
except Exception as e:
backup.log('W', f"failed to parse variables from {args.vars}: {str(e)}")

# Run backup
captured_output: typing.List[bytes] = []

if args.dry_run:
if args.debug:
backup.run(sys.stdout.buffer)
else:
with open(os.devnull, "wb") as out:
backup.run(out)
sys.stdout.flush()
else:
borg = subprocess.Popen([borg_sh,
"create",
"--verbose",
"--progress",
"--log-json",
"--list",
"--filter", "E",
"--stats",
"--checkpoint-interval", "900",
"--compression", "zstd,3",
"--paths-from-stdin",
"--paths-delimiter", "\\0",
"::" + hostname + "-{now:%Y%m%d-%H%M%S}"],
def run_borg(self, argv: typing.List[str],
stdin_writer: typing.Callable[[typing.IO[bytes]],
typing.Any]=None):
"""Run a borg command, capturing and displaying output, while feeding
input using stdin_writer. Returns True on Borg success, False on error.
"""
borg = subprocess.Popen(argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
@@ -364,7 +303,7 @@ def main(argv: typing.List[str]):
line = f"[exception: {str(e)} ]".encode() + line
sys.stdout.buffer.write(line)
sys.stdout.flush()
captured_output.append(line)
self.captured_output.append(line)
fh.close()
def _reader_thread(fh):
try:
@@ -378,9 +317,10 @@ def main(argv: typing.List[str]):
reader.start()

try:
# Give borg some time to start, just to clean up stdout
time.sleep(1)
backup.run(borg.stdin)
if stdin_writer:
# Give borg some time to start, just to clean up stdout
time.sleep(1)
stdin_writer(borg.stdin)
except BrokenPipeError:
sys.stderr.write(f"broken pipe\n")
sys.stderr.flush()
@@ -393,13 +333,100 @@ def main(argv: typing.List[str]):
reader.join()
ret = borg.returncode
if ret < 0:
backup.log('E', f"borg exited with signal {-ret}")
self.log('E', f"borg exited with signal {-ret}")
elif ret == 2 or borg_saw_errors:
backup.log('E', f"borg exited with errors (ret={ret})")
self.log('E', f"borg exited with errors (ret={ret})")
elif ret == 1 and borg_saw_warnings:
backup.log('W', f"borg exited with warnings (ret={ret})")
self.log('W', f"borg exited with warnings (ret={ret})")
elif ret != 0:
backup.log('E', f"borg exited with unknown error code {ret}")
self.log('E', f"borg exited with unknown error code {ret}")
else:
return True
return False

def main(argv: typing.List[str]):
import argparse

def humansize(string):
return humanfriendly.parse_size(string)

# Parse args
parser = argparse.ArgumentParser(
prog=argv[0],
description="Back up the local system using borg",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)

base = pathlib.Path(__file__).parent
parser.add_argument('-c', '--config',
help="Config file", default=str(base / "config.yaml"))
parser.add_argument('-v', '--vars',
help="Variables file", default=str(base / "vars.sh"))
parser.add_argument('-n', '--dry-run', action="store_true",
help="Just print log output, don't run borg")
parser.add_argument('-d', '--debug', action="store_true",
help="Print filenames for --dry-run")

args = parser.parse_args()
config = Config(args.config)
backup = Backup(config, args.dry_run)

# Parse variables from vars.sh
hostname = os.uname().nodename
borg_sh = str(base / "borg.sh")
notify_sh = str(base / "notify.sh")
try:
with open(args.vars) as f:
for line in f:
m = re.match(r"\s*export\s*([A-Z_]+)=(.*)", line)
if not m:
continue
var = m.group(1)
value = m.group(2)
if var == "HOSTNAME":
hostname = value
if var == "BORG":
borg_sh = value
if var == "BORG_DIR":
notify_sh = str(pathlib.Path(value) / "notify.sh")
except Exception as e:
backup.log('W', f"failed to parse variables from {args.vars}: {str(e)}")

# Run backup
if args.dry_run:
if args.debug:
backup.run(sys.stdout.buffer)
else:
with open(os.devnull, "wb") as out:
backup.run(out)
sys.stdout.flush()
else:
if backup.run_borg([borg_sh,
"create",
"--verbose",
"--progress",
"--log-json",
"--list",
"--filter", "E",
"--stats",
"--checkpoint-interval", "900",
"--compression", "zstd,3",
"--paths-from-stdin",
"--paths-delimiter", "\\0",
"::" + hostname + "-{now:%Y%m%d-%H%M%S}"],
stdin_writer=backup.run):
# backup success; run prune. Note that this won't actually free
# space until a "./borg.sh --rw compact", because we're in
# append-only mode.
backup.run_borg([borg_sh,
"prune",
"--verbose",
"--progress",
"--log-json",
"--stats",
"--keep-within=7d",
"--keep-daily=14",
"--keep-weekly=8",
"--keep-monthly=-1"])

# See if we had any errors
warnings = sum(1 for (letter, msg) in backup.logs if letter == 'W')
@@ -438,7 +465,7 @@ def main(argv: typing.List[str]):
body_text = "\n".join(body).encode()

# Followed by borg output
body_text += b"\n\nBorg output:\n" + b"".join(captured_output)
body_text += b"\n\nBorg output:\n" + b"".join(backup.captured_output)

# Subject summary
if errmsg and warnmsg:


Loading…
Cancel
Save