124 lines
6.4 KiB
Python
124 lines
6.4 KiB
Python
# actions/backup/run.py
|
|
"""Label-driven restic backup via resticontainer (git.ermnvldmr.com/deytenit/resticontainer).
|
|
Runs the resticontainer core service one-shot: init-if-needed -> backup (discovers
|
|
`restic.*`-labelled services, runs their pre-hooks/stops, snapshots the union of the
|
|
resolved host paths in a single restic run) -> forget --prune -> Telegram. Every
|
|
non-`backup` subcommand passes straight through to restic, so init/snapshots/forget
|
|
behave exactly as before."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
from engine.descriptor import Meta, Arg
|
|
|
|
METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).",
|
|
args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")])
|
|
TITLE = "Restic Host Backup"
|
|
|
|
def write_backup_status(path, state, timestamp):
|
|
path = Path(path)
|
|
previous = json.loads(path.read_text()) if path.exists() else {}
|
|
status = {"state": state, "updated_at": timestamp,
|
|
"last_success": timestamp if state == "success" else previous.get("last_success", 0)}
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(path.name + f".{os.getpid()}.tmp")
|
|
temporary.write_text(json.dumps(status) + "\n")
|
|
temporary.chmod(0o644)
|
|
temporary.replace(path)
|
|
|
|
def restore_containers(ctx, container_ids):
|
|
"""Recover previously running backup targets, retrying namespace dependents."""
|
|
pending = list(container_ids)
|
|
if not pending:
|
|
return
|
|
for _ in range(len(pending) + 1):
|
|
cp = ctx.sys.run(["docker", "inspect", *pending], capture=True)
|
|
stopped = [c["Id"] for c in json.loads(cp.stdout) if not c["State"]["Running"]]
|
|
if not stopped:
|
|
return
|
|
progress = False
|
|
for container_id in stopped:
|
|
cp = ctx.sys.run(["docker", "start", container_id], check=False, capture=True)
|
|
progress = progress or cp.returncode == 0
|
|
if not progress:
|
|
raise RuntimeError("Backup left containers stopped: " + ", ".join(stopped))
|
|
pending = stopped
|
|
raise RuntimeError("Backup containers did not stay running: " + ", ".join(pending))
|
|
|
|
def run(ctx, args):
|
|
log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url
|
|
core = ctx.paths.core
|
|
host = ctx.node.host # restic --host: the node's public host name (APEX_NODE_HOST)
|
|
tag = "biweekly" # restic --tag: snapshot cadence (this cron fires on the 5th & 20th)
|
|
if not os.path.isfile(os.path.join(core, "docker-compose.yml")):
|
|
log.error(f"Core compose not found in {core}"); raise SystemExit(1)
|
|
env = dict(os.environ, COMPOSE_PROFILES="manual")
|
|
env.update({k: v for k, v in ctx.vars().items() if k.startswith("APEX_")}) # tier paths, identity
|
|
envfiles = []
|
|
for ef in (".env", "apex.env"): # secrets (.env) + committed scalars (apex.env)
|
|
if os.path.isfile(os.path.join(core, ef)):
|
|
envfiles += ["--env-file", ef]
|
|
|
|
def restic(*cmd, check=True, capture=False):
|
|
return ctx.sys.run(["docker", "compose", *envfiles, "--profile", "manual",
|
|
"run", "--rm", "-T", "resticontainer", *cmd],
|
|
cwd=core, env=env, check=check, capture=capture)
|
|
|
|
def host_snapshot_ids():
|
|
"""This host's snapshot IDs, or None if it can't be determined (never fail on this)."""
|
|
cp = restic("snapshots", "--host", host, "--json", check=False, capture=True)
|
|
if cp.returncode != 0:
|
|
return None
|
|
try:
|
|
return {s.get("id") for s in json.loads(cp.stdout or "[]")}
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
status_path = Path("/var/lib/apex/backup-status.json")
|
|
write_backup_status(status_path, "running", time.time())
|
|
try:
|
|
log.info("Checking restic repository availability...")
|
|
if restic("snapshots", check=False).returncode == 0:
|
|
log.info("Restic repository reachable.")
|
|
elif restic("init", check=False).returncode != 0:
|
|
ctx.notify.error(TITLE, url, "restic init failed"); raise SystemExit(1)
|
|
before = host_snapshot_ids()
|
|
log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...")
|
|
targets = ctx.sys.run(["docker", "ps", "-q", "--no-trunc",
|
|
"--filter", "label=restic.enable=true",
|
|
"--filter", "label=restic.backup.stop=true"], capture=True).stdout.split()
|
|
try:
|
|
result = restic("backup", "--host", host, "--tag", tag, "--compression", "max",
|
|
check=False)
|
|
finally:
|
|
# Resticontainer may start a network-namespace dependent before its owner.
|
|
restore_containers(ctx, targets)
|
|
if result.returncode != 0:
|
|
ctx.notify.error(TITLE, url, "restic backup failed"); raise SystemExit(1)
|
|
# Guard: the label-driven backup is a silent no-op if no service carries the
|
|
# `restic.enable` label (resticontainer finds zero paths and skips restic).
|
|
# Prove a fresh snapshot for this host actually landed before we prune/notify.
|
|
after = host_snapshot_ids()
|
|
if before is not None and after is not None and not (after - before):
|
|
log.error("No new snapshot produced — refusing to report success.")
|
|
ctx.notify.error(TITLE, url,
|
|
"backup produced no new snapshot (no restic.* labels discovered?)")
|
|
raise SystemExit(1)
|
|
log.info("Applying retention (forget --prune)...")
|
|
if restic("forget", "--prune", "--host", host, "--tag", tag,
|
|
"--keep-daily", "7", "--keep-weekly", "4",
|
|
"--keep-monthly", "12", check=False).returncode != 0:
|
|
ctx.notify.error(TITLE, url, "restic forget/prune failed"); raise SystemExit(1)
|
|
log.success("Restic backup completed.")
|
|
if not ctx.notify.success(TITLE, url, f"Restic backup + retention succeeded for {node}"):
|
|
raise SystemExit(1) # bash exited 1 when the success telegram failed to send
|
|
write_backup_status(status_path, "success", time.time())
|
|
except SystemExit:
|
|
write_backup_status(status_path, "failed", time.time())
|
|
raise
|
|
except Exception as e:
|
|
write_backup_status(status_path, "failed", time.time())
|
|
log.error(f"{type(e).__name__}: {e}")
|
|
ctx.notify.error(TITLE, url, f"backup failed: {e}")
|
|
raise SystemExit(1) from e
|