Merge deployed native Alloy and backup recovery changes [skip ci]

actions/backup/run.py

@@ -7,12 +7,44 @@ non-`backup` subcommand passes straight through to restic, so init/snapshots/for
 behave exactly as before."""
 import json
 import os
+from pathlib import Path
+import time
 from engine.descriptor import Meta, Arg
 
 METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).",
                 args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")])
 TITLE = "Restic Host Backup"
 
+def write_backup_status(path, state, timestamp):
+    path = Path(path)
+    previous = json.loads(path.read_text()) if path.exists() else {}
+    status = {"state": state, "updated_at": timestamp,
+              "last_success": timestamp if state == "success" else previous.get("last_success", 0)}
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(path.name + f".{os.getpid()}.tmp")
+    temporary.write_text(json.dumps(status) + "\n")
+    temporary.chmod(0o644)
+    temporary.replace(path)
+
+def restore_containers(ctx, container_ids):
+    """Recover previously running backup targets, retrying namespace dependents."""
+    pending = list(container_ids)
+    if not pending:
+        return
+    for _ in range(len(pending) + 1):
+        cp = ctx.sys.run(["docker", "inspect", *pending], capture=True)
+        stopped = [c["Id"] for c in json.loads(cp.stdout) if not c["State"]["Running"]]
+        if not stopped:
+            return
+        progress = False
+        for container_id in stopped:
+            cp = ctx.sys.run(["docker", "start", container_id], check=False, capture=True)
+            progress = progress or cp.returncode == 0
+        if not progress:
+            raise RuntimeError("Backup left containers stopped: " + ", ".join(stopped))
+        pending = stopped
+    raise RuntimeError("Backup containers did not stay running: " + ", ".join(pending))
+
 def run(ctx, args):
     log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url
     core = ctx.paths.core
@@ -42,6 +74,8 @@ def run(ctx, args):
         except (ValueError, TypeError):
             return None
 
+    status_path = Path("/var/lib/apex/backup-status.json")
+    write_backup_status(status_path, "running", time.time())
     try:
         log.info("Checking restic repository availability...")
         if restic("snapshots", check=False).returncode == 0:
@@ -50,8 +84,16 @@ def run(ctx, args):
             ctx.notify.error(TITLE, url, "restic init failed"); raise SystemExit(1)
         before = host_snapshot_ids()
         log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...")
-        if restic("backup", "--host", host, "--tag", tag, "--compression", "max",
-                  check=False).returncode != 0:
+        targets = ctx.sys.run(["docker", "ps", "-q", "--no-trunc",
+                               "--filter", "label=restic.enable=true",
+                               "--filter", "label=restic.backup.stop=true"], capture=True).stdout.split()
+        try:
+            result = restic("backup", "--host", host, "--tag", tag, "--compression", "max",
+                            check=False)
+        finally:
+            # Resticontainer may start a network-namespace dependent before its owner.
+            restore_containers(ctx, targets)
+        if result.returncode != 0:
             ctx.notify.error(TITLE, url, "restic backup failed"); raise SystemExit(1)
         # Guard: the label-driven backup is a silent no-op if no service carries the
         # `restic.enable` label (resticontainer finds zero paths and skips restic).
@@ -70,9 +112,12 @@ def run(ctx, args):
         log.success("Restic backup completed.")
         if not ctx.notify.success(TITLE, url, f"Restic backup + retention succeeded for {node}"):
             raise SystemExit(1)   # bash exited 1 when the success telegram failed to send
+        write_backup_status(status_path, "success", time.time())
     except SystemExit:
+        write_backup_status(status_path, "failed", time.time())
         raise
     except Exception as e:
+        write_backup_status(status_path, "failed", time.time())
         log.error(f"{type(e).__name__}: {e}")
         ctx.notify.error(TITLE, url, f"backup failed: {e}")
         raise SystemExit(1) from e

actions/configure/alloy.py

@@ -0,0 +1,77 @@
+"""Deploy the opt-in native Alloy service using shared host/guest configs."""
+import json
+from pathlib import Path
+import re
+import tempfile
+
+from engine.descriptor import Flag, Meta
+
+METADATA = Meta(summary="Configure native Alloy from configs/alloy/config.json.",
+                args=[Flag("--dry-run", "Validate the deployment plan without changes.")])
+
+
+def deployment(ctx):
+    settings = Path(ctx.paths.configs) / "alloy/config.json"
+    if not settings.exists():
+        return None
+    values = json.loads(settings.read_text())
+    role = values.get("role")
+    if role not in ("host", "guest"):
+        raise ValueError("Alloy role must be host or guest")
+    env_file = values.get("environment_file", "")
+    if not re.fullmatch(r"/[A-Za-z0-9_./-]+", env_file):
+        raise ValueError("Alloy environment_file must be an absolute path without whitespace")
+    source = Path(ctx.paths.commons) / "etc/alloy" / (
+        "native-host.alloy" if role == "host" else "native.alloy")
+    config = source.read_text()
+    extra = Path(ctx.paths.configs) / "alloy/extra.alloy"
+    if extra.is_file():
+        config += "\n" + extra.read_text()
+    unit = ("[Service]\nEnvironmentFile=" + env_file + "\n"
+            'ExecStart=\nExecStart=/usr/bin/alloy run --storage.path=/var/lib/alloy --server.http.listen-addr=127.0.0.1:12346 /etc/alloy/config.alloy\n'
+            + ("User=root\nGroup=root\n" if role == "guest" else "") +
+            "SupplementaryGroups=systemd-journal" + (" docker" if role == "guest" else "") + "\n")
+    return config, unit, env_file
+
+
+def run(ctx, args):
+    plan = deployment(ctx)
+    if plan is None:
+        ctx.log.info("Native Alloy is not configured for this node; skipping.")
+        return
+    config, unit, env_file = plan
+    if args.dry_run:
+        ctx.log.info("Native Alloy deployment plan validated; no changes made.")
+        return
+    ctx.sys.sudo(["test", "-f", env_file])
+    # Validate with the actual service environment without printing credentials.
+    validator = '''import os, stat, subprocess, sys
+from pathlib import Path
+# Environment files deliberately use plain KEY=value, with no shell evaluation.
+secret = Path(sys.argv[1])
+info = secret.stat()
+if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077:
+    raise RuntimeError('Alloy environment file must be root-owned with mode 0600 or stricter')
+for line in secret.read_text().splitlines():
+    if not line.strip() or line.lstrip().startswith('#'): continue
+    key, value = line.split('=', 1)
+    os.environ[key] = value
+subprocess.run(['alloy', 'validate', sys.argv[2]], check=True)
+'''
+    with tempfile.TemporaryDirectory(prefix="apex-alloy-") as directory:
+        config_path = Path(directory) / "config.alloy"
+        unit_path = Path(directory) / "apex.conf"
+        config_path.write_text(config)
+        unit_path.write_text(unit)
+        ctx.sys.sudo(["python3", "-c", validator, env_file, str(config_path)])
+        ctx.sys.sudo(["install", "-d", "-m", "0755", "/etc/alloy",
+                      "/etc/systemd/system/alloy.service.d"])
+        ctx.sys.sudo(["install", "-m", "0644", str(config_path), "/etc/alloy/config.alloy"])
+        ctx.sys.sudo(["install", "-m", "0644", str(unit_path),
+                      "/etc/systemd/system/alloy.service.d/apex.conf"])
+    ctx.sys.sudo(["systemctl", "unmask", "alloy.service"])
+    ctx.sys.daemon_reload()
+    ctx.sys.sudo(["systemctl", "enable", "alloy.service"])
+    ctx.sys.sudo(["systemctl", "restart", "alloy.service"])
+    ctx.sys.sudo(["systemctl", "is-active", "--quiet", "alloy.service"])
+    ctx.log.success("Configured native Alloy.")