Deliver tested native Alloy and backup recovery sources

actions/backup/run.py

@@ -7,12 +7,44 @@ non-`backup` subcommand passes straight through to restic, so init/snapshots/for
 behave exactly as before."""
 import json
 import os
+from pathlib import Path
+import time
 from engine.descriptor import Meta, Arg
 
 METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).",
                 args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")])
 TITLE = "Restic Host Backup"
 
+def write_backup_status(path, state, timestamp):
+    path = Path(path)
+    previous = json.loads(path.read_text()) if path.exists() else {}
+    status = {"state": state, "updated_at": timestamp,
+              "last_success": timestamp if state == "success" else previous.get("last_success", 0)}
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(path.name + f".{os.getpid()}.tmp")
+    temporary.write_text(json.dumps(status) + "\n")
+    temporary.chmod(0o644)
+    temporary.replace(path)
+
+def restore_containers(ctx, container_ids):
+    """Recover previously running backup targets, retrying namespace dependents."""
+    pending = list(container_ids)
+    if not pending:
+        return
+    for _ in range(len(pending) + 1):
+        cp = ctx.sys.run(["docker", "inspect", *pending], capture=True)
+        stopped = [c["Id"] for c in json.loads(cp.stdout) if not c["State"]["Running"]]
+        if not stopped:
+            return
+        progress = False
+        for container_id in stopped:
+            cp = ctx.sys.run(["docker", "start", container_id], check=False, capture=True)
+            progress = progress or cp.returncode == 0
+        if not progress:
+            raise RuntimeError("Backup left containers stopped: " + ", ".join(stopped))
+        pending = stopped
+    raise RuntimeError("Backup containers did not stay running: " + ", ".join(pending))
+
 def run(ctx, args):
     log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url
     core = ctx.paths.core
@@ -42,6 +74,8 @@ def run(ctx, args):
         except (ValueError, TypeError):
             return None
 
+    status_path = Path("/var/lib/apex/backup-status.json")
+    write_backup_status(status_path, "running", time.time())
     try:
         log.info("Checking restic repository availability...")
         if restic("snapshots", check=False).returncode == 0:
@@ -50,8 +84,16 @@ def run(ctx, args):
             ctx.notify.error(TITLE, url, "restic init failed"); raise SystemExit(1)
         before = host_snapshot_ids()
         log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...")
-        if restic("backup", "--host", host, "--tag", tag, "--compression", "max",
-                  check=False).returncode != 0:
+        targets = ctx.sys.run(["docker", "ps", "-q", "--no-trunc",
+                               "--filter", "label=restic.enable=true",
+                               "--filter", "label=restic.backup.stop=true"], capture=True).stdout.split()
+        try:
+            result = restic("backup", "--host", host, "--tag", tag, "--compression", "max",
+                            check=False)
+        finally:
+            # Resticontainer may start a network-namespace dependent before its owner.
+            restore_containers(ctx, targets)
+        if result.returncode != 0:
             ctx.notify.error(TITLE, url, "restic backup failed"); raise SystemExit(1)
         # Guard: the label-driven backup is a silent no-op if no service carries the
         # `restic.enable` label (resticontainer finds zero paths and skips restic).
@@ -70,9 +112,12 @@ def run(ctx, args):
         log.success("Restic backup completed.")
         if not ctx.notify.success(TITLE, url, f"Restic backup + retention succeeded for {node}"):
             raise SystemExit(1)   # bash exited 1 when the success telegram failed to send
+        write_backup_status(status_path, "success", time.time())
     except SystemExit:
+        write_backup_status(status_path, "failed", time.time())
         raise
     except Exception as e:
+        write_backup_status(status_path, "failed", time.time())
         log.error(f"{type(e).__name__}: {e}")
         ctx.notify.error(TITLE, url, f"backup failed: {e}")
-        raise SystemExit(1)
+        raise SystemExit(1) from e