From dd9b4fb93b8239612dcc5c3f3286a4228c7d53ff Mon Sep 17 00:00:00 2001 From: Vladimir Eremin Date: Thu, 5 Nov 2026 02:14:18 +0300 Subject: [PATCH] Deliver tested native Alloy and backup recovery sources --- actions/backup/run.py | 51 ++++++++++++++++++++++++-- ci/packer/apex-debian-13.pkr.hcl | 7 ++-- ci/packer/scripts/02-alloy-package.sh | 18 ++++++++++ tests/test_backup_run.py | 49 +++++++++++++++++++++++++ tests/test_configure_alloy.py | 52 +++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 ci/packer/scripts/02-alloy-package.sh create mode 100644 tests/test_backup_run.py create mode 100644 tests/test_configure_alloy.py diff --git a/actions/backup/run.py b/actions/backup/run.py index 621b090..71be03c 100644 --- a/actions/backup/run.py +++ b/actions/backup/run.py @@ -7,12 +7,44 @@ non-`backup` subcommand passes straight through to restic, so init/snapshots/for behave exactly as before.""" import json import os +from pathlib import Path +import time from engine.descriptor import Meta, Arg METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).", args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")]) TITLE = "Restic Host Backup" +def write_backup_status(path, state, timestamp): + path = Path(path) + previous = json.loads(path.read_text()) if path.exists() else {} + status = {"state": state, "updated_at": timestamp, + "last_success": timestamp if state == "success" else previous.get("last_success", 0)} + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + f".{os.getpid()}.tmp") + temporary.write_text(json.dumps(status) + "\n") + temporary.chmod(0o644) + temporary.replace(path) + +def restore_containers(ctx, container_ids): + """Recover previously running backup targets, retrying namespace dependents.""" + pending = list(container_ids) + if not pending: + return + for _ in range(len(pending) + 1): + cp = ctx.sys.run(["docker", "inspect", *pending], capture=True) + stopped = [c["Id"] for c in json.loads(cp.stdout) if not c["State"]["Running"]] + if not stopped: + return + progress = False + for container_id in stopped: + cp = ctx.sys.run(["docker", "start", container_id], check=False, capture=True) + progress = progress or cp.returncode == 0 + if not progress: + raise RuntimeError("Backup left containers stopped: " + ", ".join(stopped)) + pending = stopped + raise RuntimeError("Backup containers did not stay running: " + ", ".join(pending)) + def run(ctx, args): log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url core = ctx.paths.core @@ -42,6 +74,8 @@ def run(ctx, args): except (ValueError, TypeError): return None + status_path = Path("/var/lib/apex/backup-status.json") + write_backup_status(status_path, "running", time.time()) try: log.info("Checking restic repository availability...") if restic("snapshots", check=False).returncode == 0: @@ -50,8 +84,16 @@ def run(ctx, args): ctx.notify.error(TITLE, url, "restic init failed"); raise SystemExit(1) before = host_snapshot_ids() log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...") - if restic("backup", "--host", host, "--tag", tag, "--compression", "max", - check=False).returncode != 0: + targets = ctx.sys.run(["docker", "ps", "-q", "--no-trunc", + "--filter", "label=restic.enable=true", + "--filter", "label=restic.backup.stop=true"], capture=True).stdout.split() + try: + result = restic("backup", "--host", host, "--tag", tag, "--compression", "max", + check=False) + finally: + # Resticontainer may start a network-namespace dependent before its owner. + restore_containers(ctx, targets) + if result.returncode != 0: ctx.notify.error(TITLE, url, "restic backup failed"); raise SystemExit(1) # Guard: the label-driven backup is a silent no-op if no service carries the # `restic.enable` label (resticontainer finds zero paths and skips restic). @@ -70,9 +112,12 @@ def run(ctx, args): log.success("Restic backup completed.") if not ctx.notify.success(TITLE, url, f"Restic backup + retention succeeded for {node}"): raise SystemExit(1) # bash exited 1 when the success telegram failed to send + write_backup_status(status_path, "success", time.time()) except SystemExit: + write_backup_status(status_path, "failed", time.time()) raise except Exception as e: + write_backup_status(status_path, "failed", time.time()) log.error(f"{type(e).__name__}: {e}") ctx.notify.error(TITLE, url, f"backup failed: {e}") - raise SystemExit(1) + raise SystemExit(1) from e diff --git a/ci/packer/apex-debian-13.pkr.hcl b/ci/packer/apex-debian-13.pkr.hcl index b36b8e0..83314fb 100644 --- a/ci/packer/apex-debian-13.pkr.hcl +++ b/ci/packer/apex-debian-13.pkr.hcl @@ -61,7 +61,7 @@ build { sources = ["source.qemu.apex_debian"] provisioner "file" { - source = "engine" + source = "ci/scripts/init-node/provisioning" destination = "/tmp/" } @@ -69,8 +69,8 @@ build { execute_command = "sudo -n sh -c '{{ .Vars }} {{ .Path }}'" environment_vars = ["APEX_PROVISION_REVISION=${var.provision_revision}"] inline = [ - "bash /tmp/engine/provisioning/seed.sh", - "cd /tmp && python3 -m engine.provisioning.image install" + "bash /tmp/provisioning/seed.sh", + "cd /tmp && python3 -m provisioning.image install" ] } @@ -79,6 +79,7 @@ build { scripts = [ "ci/packer/scripts/01-base-system.sh", "ci/packer/scripts/02-storage-docker.sh", + "ci/packer/scripts/02-alloy-package.sh", "ci/packer/scripts/03-bootstrap-helper.sh", "ci/packer/scripts/04-cleanup.sh" ] diff --git a/ci/packer/scripts/02-alloy-package.sh b/ci/packer/scripts/02-alloy-package.sh new file mode 100644 index 0000000..563ce8e --- /dev/null +++ b/ci/packer/scripts/02-alloy-package.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Service configuration and credentials are deployed later by apex configure/alloy. +alloy_version=1.21.3 +alloy_sha256=dbbd705de8ab37dec0cbd6361a3fc7e754d6b2854b57ce672a83e6366199ba73 +[[ "$(dpkg --print-architecture)" == amd64 ]] +alloy_deb=$(mktemp --suffix=.deb) +trap 'rm -f "$alloy_deb"' EXIT +curl --fail --location --retry 3 --output "$alloy_deb" \ + "https://github.com/grafana/alloy/releases/download/v${alloy_version}/alloy-${alloy_version}-1.amd64.deb" +printf '%s %s\n' "$alloy_sha256" "$alloy_deb" | sha256sum --check --status +systemctl mask alloy.service +DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$alloy_deb" +systemctl disable --now alloy.service +systemctl unmask alloy.service +! systemctl is-active --quiet alloy.service +! systemctl is-enabled --quiet alloy.service diff --git a/tests/test_backup_run.py b/tests/test_backup_run.py new file mode 100644 index 0000000..61a8b54 --- /dev/null +++ b/tests/test_backup_run.py @@ -0,0 +1,49 @@ +import json +from pathlib import Path +from types import SimpleNamespace +import unittest +import tempfile +from unittest.mock import Mock + +from engine import overlay + +backup = overlay.load_module(str(Path(__file__).resolve().parents[1] / 'actions/backup/run.py'), + 'test_backup_action') + + +class BackupRecovery(unittest.TestCase): + def test_failed_attempt_preserves_last_success_and_records_new_failure(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'status.json' + backup.write_backup_status(path, 'success', 100) + backup.write_backup_status(path, 'running', 200) + backup.write_backup_status(path, 'failed', 210) + status = json.loads(path.read_text()) + self.assertEqual(status['last_success'], 100) + self.assertEqual(status['state'], 'failed') + self.assertEqual(status['updated_at'], 210) + + def test_recovers_namespace_owner_before_dependent_without_starting_other_containers(self): + running = {'komf': False, 'komga': False, 'intentionally-stopped': False} + def docker(command, **kwargs): + if command[1] == 'inspect': + return SimpleNamespace(returncode=0, stdout=json.dumps([ + {'Id': name, 'State': {'Running': running[name]}} for name in command[2:]])) + name = command[2] + if name == 'komf' and not running['komga']: + return SimpleNamespace(returncode=1) + running[name] = True + return SimpleNamespace(returncode=0) + ctx = SimpleNamespace(sys=SimpleNamespace(run=docker), log=Mock()) + backup.restore_containers(ctx, ['komf', 'komga']) + self.assertTrue(running['komf']) + self.assertTrue(running['komga']) + self.assertFalse(running['intentionally-stopped']) + + def test_unrecoverable_container_prevents_success(self): + def docker(command, **kwargs): + return SimpleNamespace(returncode=0 if command[1] == 'inspect' else 1, + stdout='[{"Id":"broken","State":{"Running":false}}]') + ctx = SimpleNamespace(sys=SimpleNamespace(run=docker), log=Mock()) + with self.assertRaisesRegex(RuntimeError, 'broken'): + backup.restore_containers(ctx, ['broken']) diff --git a/tests/test_configure_alloy.py b/tests/test_configure_alloy.py new file mode 100644 index 0000000..560ed18 --- /dev/null +++ b/tests/test_configure_alloy.py @@ -0,0 +1,52 @@ +"""Native Alloy opt-in and validation boundaries.""" +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock + +from engine import overlay + +ROOT = Path(__file__).resolve().parents[1] +alloy = overlay.load_module(str(ROOT / 'actions/configure/alloy.py'), 'test_alloy_action') + + +class NativeAlloy(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.configs = Path(self.temp.name) + self.ctx = SimpleNamespace(paths=SimpleNamespace(configs=str(self.configs), commons=str(ROOT)), + sys=Mock(), log=Mock()) + + def configure(self, **settings): + (self.configs / 'alloy').mkdir(exist_ok=True) + (self.configs / 'alloy/config.json').write_text(json.dumps(settings)) + + def test_unconfigured_node_is_untouched(self): + alloy.run(self.ctx, SimpleNamespace(dry_run=False)) + self.ctx.sys.sudo.assert_not_called() + + def test_invalid_role_or_environment_path_cannot_change_service(self): + for role, path in [('unknown', '/etc/alloy/secrets'), ('host', '/tmp/file\nUser=root')]: + self.configure(role=role, environment_file=path) + with self.assertRaises(ValueError): + alloy.run(self.ctx, SimpleNamespace(dry_run=False)) + self.ctx.sys.sudo.assert_not_called() + + def test_dry_run_never_installs_or_starts(self): + self.configure(role='guest', environment_file='/etc/alloy/secrets') + alloy.run(self.ctx, SimpleNamespace(dry_run=True)) + self.ctx.sys.sudo.assert_not_called() + + def test_failed_config_validation_leaves_existing_service_untouched(self): + self.configure(role='host', environment_file='/etc/alloy/secrets') + def execute(command): + if command[0] == 'python3': + raise RuntimeError('invalid Alloy configuration') + self.assertEqual(command, ['test', '-f', '/etc/alloy/secrets']) + self.ctx.sys.sudo.side_effect = execute + with self.assertRaisesRegex(RuntimeError, 'invalid Alloy'): + alloy.run(self.ctx, SimpleNamespace(dry_run=False)) + self.ctx.sys.daemon_reload.assert_not_called()