Deliver tested native Alloy and backup recovery sources

actions/backup/run.py

@@ -7,12 +7,44 @@ non-`backup` subcommand passes straight through to restic, so init/snapshots/for
 behave exactly as before."""
 import json
 import os
+from pathlib import Path
+import time
 from engine.descriptor import Meta, Arg
 
 METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).",
                 args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")])
 TITLE = "Restic Host Backup"
 
+def write_backup_status(path, state, timestamp):
+    path = Path(path)
+    previous = json.loads(path.read_text()) if path.exists() else {}
+    status = {"state": state, "updated_at": timestamp,
+              "last_success": timestamp if state == "success" else previous.get("last_success", 0)}
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(path.name + f".{os.getpid()}.tmp")
+    temporary.write_text(json.dumps(status) + "\n")
+    temporary.chmod(0o644)
+    temporary.replace(path)
+
+def restore_containers(ctx, container_ids):
+    """Recover previously running backup targets, retrying namespace dependents."""
+    pending = list(container_ids)
+    if not pending:
+        return
+    for _ in range(len(pending) + 1):
+        cp = ctx.sys.run(["docker", "inspect", *pending], capture=True)
+        stopped = [c["Id"] for c in json.loads(cp.stdout) if not c["State"]["Running"]]
+        if not stopped:
+            return
+        progress = False
+        for container_id in stopped:
+            cp = ctx.sys.run(["docker", "start", container_id], check=False, capture=True)
+            progress = progress or cp.returncode == 0
+        if not progress:
+            raise RuntimeError("Backup left containers stopped: " + ", ".join(stopped))
+        pending = stopped
+    raise RuntimeError("Backup containers did not stay running: " + ", ".join(pending))
+
 def run(ctx, args):
     log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url
     core = ctx.paths.core
@@ -42,6 +74,8 @@ def run(ctx, args):
         except (ValueError, TypeError):
             return None
 
+    status_path = Path("/var/lib/apex/backup-status.json")
+    write_backup_status(status_path, "running", time.time())
     try:
         log.info("Checking restic repository availability...")
         if restic("snapshots", check=False).returncode == 0:
@@ -50,8 +84,16 @@ def run(ctx, args):
             ctx.notify.error(TITLE, url, "restic init failed"); raise SystemExit(1)
         before = host_snapshot_ids()
         log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...")
-        if restic("backup", "--host", host, "--tag", tag, "--compression", "max",
-                  check=False).returncode != 0:
+        targets = ctx.sys.run(["docker", "ps", "-q", "--no-trunc",
+                               "--filter", "label=restic.enable=true",
+                               "--filter", "label=restic.backup.stop=true"], capture=True).stdout.split()
+        try:
+            result = restic("backup", "--host", host, "--tag", tag, "--compression", "max",
+                            check=False)
+        finally:
+            # Resticontainer may start a network-namespace dependent before its owner.
+            restore_containers(ctx, targets)
+        if result.returncode != 0:
             ctx.notify.error(TITLE, url, "restic backup failed"); raise SystemExit(1)
         # Guard: the label-driven backup is a silent no-op if no service carries the
         # `restic.enable` label (resticontainer finds zero paths and skips restic).
@@ -70,9 +112,12 @@ def run(ctx, args):
         log.success("Restic backup completed.")
         if not ctx.notify.success(TITLE, url, f"Restic backup + retention succeeded for {node}"):
             raise SystemExit(1)   # bash exited 1 when the success telegram failed to send
+        write_backup_status(status_path, "success", time.time())
     except SystemExit:
+        write_backup_status(status_path, "failed", time.time())
         raise
     except Exception as e:
+        write_backup_status(status_path, "failed", time.time())
         log.error(f"{type(e).__name__}: {e}")
         ctx.notify.error(TITLE, url, f"backup failed: {e}")
-        raise SystemExit(1)
+        raise SystemExit(1) from e

ci/packer/apex-debian-13.pkr.hcl

@@ -61,7 +61,7 @@ build {
   sources = ["source.qemu.apex_debian"]
 
   provisioner "file" {
-    source      = "engine"
+    source      = "ci/scripts/init-node/provisioning"
     destination = "/tmp/"
   }
 
@@ -69,8 +69,8 @@ build {
     execute_command = "sudo -n sh -c '{{ .Vars }} {{ .Path }}'"
     environment_vars = ["APEX_PROVISION_REVISION=${var.provision_revision}"]
     inline = [
-      "bash /tmp/engine/provisioning/seed.sh",
-      "cd /tmp && python3 -m engine.provisioning.image install"
+      "bash /tmp/provisioning/seed.sh",
+      "cd /tmp && python3 -m provisioning.image install"
     ]
   }
 
@@ -79,6 +79,7 @@ build {
     scripts = [
       "ci/packer/scripts/01-base-system.sh",
       "ci/packer/scripts/02-storage-docker.sh",
+      "ci/packer/scripts/02-alloy-package.sh",
       "ci/packer/scripts/03-bootstrap-helper.sh",
       "ci/packer/scripts/04-cleanup.sh"
     ]

ci/packer/scripts/02-alloy-package.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Service configuration and credentials are deployed later by apex configure/alloy.
+alloy_version=1.21.3
+alloy_sha256=dbbd705de8ab37dec0cbd6361a3fc7e754d6b2854b57ce672a83e6366199ba73
+[[ "$(dpkg --print-architecture)" == amd64 ]]
+alloy_deb=$(mktemp --suffix=.deb)
+trap 'rm -f "$alloy_deb"' EXIT
+curl --fail --location --retry 3 --output "$alloy_deb" \
+  "https://github.com/grafana/alloy/releases/download/v${alloy_version}/alloy-${alloy_version}-1.amd64.deb"
+printf '%s  %s\n' "$alloy_sha256" "$alloy_deb" | sha256sum --check --status
+systemctl mask alloy.service
+DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$alloy_deb"
+systemctl disable --now alloy.service
+systemctl unmask alloy.service
+! systemctl is-active --quiet alloy.service
+! systemctl is-enabled --quiet alloy.service

tests/test_backup_run.py

@@ -0,0 +1,49 @@
+import json
+from pathlib import Path
+from types import SimpleNamespace
+import unittest
+import tempfile
+from unittest.mock import Mock
+
+from engine import overlay
+
+backup = overlay.load_module(str(Path(__file__).resolve().parents[1] / 'actions/backup/run.py'),
+                             'test_backup_action')
+
+
+class BackupRecovery(unittest.TestCase):
+    def test_failed_attempt_preserves_last_success_and_records_new_failure(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'status.json'
+            backup.write_backup_status(path, 'success', 100)
+            backup.write_backup_status(path, 'running', 200)
+            backup.write_backup_status(path, 'failed', 210)
+            status = json.loads(path.read_text())
+            self.assertEqual(status['last_success'], 100)
+            self.assertEqual(status['state'], 'failed')
+            self.assertEqual(status['updated_at'], 210)
+
+    def test_recovers_namespace_owner_before_dependent_without_starting_other_containers(self):
+        running = {'komf': False, 'komga': False, 'intentionally-stopped': False}
+        def docker(command, **kwargs):
+            if command[1] == 'inspect':
+                return SimpleNamespace(returncode=0, stdout=json.dumps([
+                    {'Id': name, 'State': {'Running': running[name]}} for name in command[2:]]))
+            name = command[2]
+            if name == 'komf' and not running['komga']:
+                return SimpleNamespace(returncode=1)
+            running[name] = True
+            return SimpleNamespace(returncode=0)
+        ctx = SimpleNamespace(sys=SimpleNamespace(run=docker), log=Mock())
+        backup.restore_containers(ctx, ['komf', 'komga'])
+        self.assertTrue(running['komf'])
+        self.assertTrue(running['komga'])
+        self.assertFalse(running['intentionally-stopped'])
+
+    def test_unrecoverable_container_prevents_success(self):
+        def docker(command, **kwargs):
+            return SimpleNamespace(returncode=0 if command[1] == 'inspect' else 1,
+                                   stdout='[{"Id":"broken","State":{"Running":false}}]')
+        ctx = SimpleNamespace(sys=SimpleNamespace(run=docker), log=Mock())
+        with self.assertRaisesRegex(RuntimeError, 'broken'):
+            backup.restore_containers(ctx, ['broken'])

tests/test_configure_alloy.py

@@ -0,0 +1,52 @@
+"""Native Alloy opt-in and validation boundaries."""
+import json
+from pathlib import Path
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import Mock
+
+from engine import overlay
+
+ROOT = Path(__file__).resolve().parents[1]
+alloy = overlay.load_module(str(ROOT / 'actions/configure/alloy.py'), 'test_alloy_action')
+
+
+class NativeAlloy(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.configs = Path(self.temp.name)
+        self.ctx = SimpleNamespace(paths=SimpleNamespace(configs=str(self.configs), commons=str(ROOT)),
+                                   sys=Mock(), log=Mock())
+
+    def configure(self, **settings):
+        (self.configs / 'alloy').mkdir(exist_ok=True)
+        (self.configs / 'alloy/config.json').write_text(json.dumps(settings))
+
+    def test_unconfigured_node_is_untouched(self):
+        alloy.run(self.ctx, SimpleNamespace(dry_run=False))
+        self.ctx.sys.sudo.assert_not_called()
+
+    def test_invalid_role_or_environment_path_cannot_change_service(self):
+        for role, path in [('unknown', '/etc/alloy/secrets'), ('host', '/tmp/file\nUser=root')]:
+            self.configure(role=role, environment_file=path)
+            with self.assertRaises(ValueError):
+                alloy.run(self.ctx, SimpleNamespace(dry_run=False))
+            self.ctx.sys.sudo.assert_not_called()
+
+    def test_dry_run_never_installs_or_starts(self):
+        self.configure(role='guest', environment_file='/etc/alloy/secrets')
+        alloy.run(self.ctx, SimpleNamespace(dry_run=True))
+        self.ctx.sys.sudo.assert_not_called()
+
+    def test_failed_config_validation_leaves_existing_service_untouched(self):
+        self.configure(role='host', environment_file='/etc/alloy/secrets')
+        def execute(command):
+            if command[0] == 'python3':
+                raise RuntimeError('invalid Alloy configuration')
+            self.assertEqual(command, ['test', '-f', '/etc/alloy/secrets'])
+        self.ctx.sys.sudo.side_effect = execute
+        with self.assertRaisesRegex(RuntimeError, 'invalid Alloy'):
+            alloy.run(self.ctx, SimpleNamespace(dry_run=False))
+        self.ctx.sys.daemon_reload.assert_not_called()