Remove the image build and consolidate operational code under scripts

scripts/engine/cli.py

@@ -0,0 +1,79 @@
+# scripts/engine/cli.py
+"""apex CLI: list / dispatch / help. Acts on the local checkout (no <node> arg)."""
+from __future__ import annotations
+import os
+import socket
+import sys
+from pathlib import Path
+
+from engine import overlay
+from engine.identity import resolve
+from engine.context import Context, Paths
+from engine.descriptor import build_parser, Meta
+
+
+def _resolve_layout():
+    commons_dir = str(Path(__file__).resolve().parents[2])
+    parent = os.path.dirname(commons_dir)
+    cwd = os.getcwd()
+
+    if os.path.isfile(os.path.join(parent, "node.env")):
+        repo_root = parent
+        prop_actions = os.path.join(repo_root, "proprietaries", "actions")
+        return commons_dir, repo_root, prop_actions, True
+    if os.path.isfile(os.path.join(cwd, "node.env")):
+        repo_root = cwd
+        prop_actions = os.path.join(repo_root, "proprietaries", "actions")
+        return commons_dir, repo_root, prop_actions, True
+    return commons_dir, commons_dir, None, False
+
+
+def _print_list(commons_actions, prop_actions):
+    print("Usage: apex <entity>/<action> [args...]\n\nAvailable actions:")
+    tag = {"commons": "", "local": "  (local)", "override": "  (local override)"}
+    for i in overlay.discover(commons_actions, prop_actions):
+        mark = f"  [DISABLED: {i.disabled}]" if i.disabled else ""
+        print(f"  {i.name:<34} {i.summary}{tag[i.source]}{mark}")
+
+
+def main(argv) -> int:
+    commons_dir, repo_root, prop_actions, is_node = _resolve_layout()
+    commons_actions = os.path.join(commons_dir, "actions")
+
+    if not argv or argv[0] in ("--help", "-h"):
+        _print_list(commons_actions, prop_actions)
+        return 0
+
+    name, rest = argv[0], argv[1:]
+    if name == "compose":
+        name = "composition/run"
+    r = overlay.resolve(name, commons_actions, prop_actions)
+    if r is None:
+        sys.stderr.write(f"[global] [core] [ERROR] Unknown action: {name}\n")
+        return 1
+
+    node, warns = (resolve(socket.getfqdn(), repo_root) if is_node else (None, []))
+    ctx = Context(node, Paths(repo_root, commons_dir), name, shadowed=r.shadowed)
+    for w in warns:
+        ctx.log.warn(w)
+
+    modname = "apex_action_" + name.replace("/", "_").replace("-", "_")
+    mod = overlay.load_module(r.path, modname)
+
+    disabled = getattr(mod, "DISABLED", None)
+    if disabled:
+        reason = disabled if isinstance(disabled, str) else "disabled"
+        ctx.log.error(f"Action {name} is disabled: {reason}")
+        return 1
+
+    meta = getattr(mod, "METADATA", Meta(summary=""))
+    args = build_parser(meta, f"apex {name}").parse_args(rest)  # --help & EX_USAGE handled here
+
+    try:
+        rc = mod.run(ctx, args)
+        return int(rc) if isinstance(rc, int) else 0
+    except SystemExit as e:
+        return e.code if isinstance(e.code, int) else 1
+    except Exception as e:  # map to general error, log with context prefix
+        ctx.log.error(f"{type(e).__name__}: {e}")
+        return 1

scripts/engine/context.py

@@ -0,0 +1,113 @@
+# scripts/engine/context.py
+"""Context and filesystem paths available to actions."""
+from __future__ import annotations
+import os
+
+from engine.lib.log import Log
+from engine.lib.sys import Sys
+from engine.lib.tpl import Tpl
+from engine.lib.host import Host
+from engine.lib.ufw import Ufw
+from engine.lib.crowdsec import Crowdsec
+from engine.lib.systemd import Systemd
+from engine.lib.notify import Notify
+
+
+class Paths:
+    def __init__(self, repo_root: str, commons_dir: str):
+        self.repo_root = repo_root
+        self.commons = commons_dir
+        self.proprietaries = os.path.join(repo_root, "proprietaries")
+        self.configs = os.path.join(repo_root, "configs")
+        self.compositions = os.path.join(repo_root, "compositions")
+        self.core = os.path.join(self.compositions, "apex")
+
+    def _tier(self, n: int) -> str:
+        return os.path.realpath(os.path.join(self.core, f"@tier{n}"))
+
+    def _stier(self, n: int) -> str:
+        return os.path.realpath(os.path.join(self.core, f"@tier{n}", "shared"))
+
+    def _root_tier(self, n: int) -> str:
+        return os.path.realpath(os.path.join(self.compositions, f"@tier{n}"))
+
+    @property
+    def root_tier1(self): return self._root_tier(1)
+    @property
+    def root_tier2(self): return self._root_tier(2)
+    @property
+    def root_tier3(self): return self._root_tier(3)
+
+    @property
+    def tier1(self): return self._tier(1)
+    @property
+    def tier2(self): return self._tier(2)
+    @property
+    def tier3(self): return self._tier(3)
+    @property
+    def stier1(self): return self._stier(1)
+    @property
+    def stier2(self): return self._stier(2)
+    @property
+    def stier3(self): return self._stier(3)
+
+
+class _CommonsWrap:
+    """ctx.commons — lazily loads and runs the shadowed commons action (opt-in wrap)."""
+    def __init__(self, ctx, shadowed_path):
+        self._ctx = ctx
+        self._path = shadowed_path
+        self._mod = None
+
+    def run(self, args):
+        if not self._path:
+            raise RuntimeError("ctx.commons is only available when a commons action is shadowed")
+        from engine import overlay
+        if self._mod is None:
+            self._mod = overlay.load_module(self._path, "apex_commons_shadowed")
+        return self._mod.run(self._ctx, args)
+
+
+class Context:
+    def __init__(self, node, paths: Paths, action_name: str, shadowed=None):
+        self.node = node
+        self.paths = paths
+        self.action = action_name
+        name = node.name if node else "global"
+        self.log = Log(name, action_name)
+        self.sys = Sys(self.log)
+        self.tpl = Tpl(paths.configs, self.log)
+        self.host = Host(self.log, self.sys)
+        self.ufw = Ufw(self.log, self.sys, self.host)
+        self.crowdsec = Crowdsec(self.log, self.sys, self.tpl, self.host)
+        self.systemd = Systemd(self.log, self.sys, self.tpl, self.host)
+        self.notify = Notify(self.log, node.host if node else "global")
+        self._shadowed = shadowed
+        self._commons = None
+
+    @property
+    def commons(self) -> _CommonsWrap:
+        if self._commons is None:
+            self._commons = _CommonsWrap(self, self._shadowed)
+        return self._commons
+
+    def vars(self) -> dict:
+        """Substitution variable set for templating: os.environ + apex identity/paths."""
+        v = dict(os.environ)
+        v["APEX_COMMONS"] = self.paths.commons        # absolute launcher dir (for crontab/systemd)
+        v["APEX_REPO_ROOT"] = self.paths.repo_root
+        if self.node:
+            v.update({
+                "APEX_NODE_HOST": self.node.host,
+                "APEX_NODE_FQDN": self.node.fqdn,
+                "APEX_SUBNET": self.node.subnet,
+                "APEX_TIER1": self.paths.tier1, "APEX_TIER2": self.paths.tier2,
+                "APEX_TIER3": self.paths.tier3,
+                "APEX_TIER1_SHARED": self.paths.stier1,
+                "APEX_TIER2_SHARED": self.paths.stier2,
+                "APEX_TIER3_SHARED": self.paths.stier3,
+                "APEX_TIER_ROOT1": self.paths.root_tier1,
+                "APEX_TIER_ROOT2": self.paths.root_tier2,
+                "APEX_TIER_ROOT3": self.paths.root_tier3,
+            })
+        return v

scripts/engine/descriptor.py

@@ -0,0 +1,77 @@
+# scripts/engine/descriptor.py
+"""Declarative action descriptors — successor to the bash action_metadata() heredoc.
+
+Meta drives BOTH the help text and argparse parsing. Exit code 64 (EX_USAGE) on a
+usage error, matching the bash engine.
+"""
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass, field
+from typing import List
+
+
+class _Parser(argparse.ArgumentParser):
+    """ArgumentParser that exits 64 (EX_USAGE) instead of 2 on a usage error."""
+
+    def error(self, message: str):
+        self.print_usage()
+        self.exit(64, f"{self.prog}: error: {message}\n")
+
+
+@dataclass
+class Arg:
+    """A positional argument. required=False -> optional positional (default None)."""
+    name: str
+    help: str = ""
+    required: bool = True
+
+    def add_to(self, p: argparse.ArgumentParser) -> None:
+        if self.required:
+            p.add_argument(self.name, help=self.help)
+        else:
+            p.add_argument(self.name, nargs="?", default=None, help=self.help)
+
+
+@dataclass
+class Opt:
+    """An optional value flag, e.g. --tier PATH, with a default."""
+    name: str
+    help: str = ""
+    default: object = None
+
+    def add_to(self, p: argparse.ArgumentParser) -> None:
+        p.add_argument(self.name, default=self.default, help=self.help)
+
+
+@dataclass
+class Flag:
+    """A boolean flag, e.g. --dry-run (store_true)."""
+    name: str
+    help: str = ""
+
+    def add_to(self, p: argparse.ArgumentParser) -> None:
+        p.add_argument(self.name, action="store_true", help=self.help)
+
+
+@dataclass
+class Rest:
+    """Greedy passthrough of all remaining argv (argparse.REMAINDER)."""
+    name: str
+    help: str = ""
+
+    def add_to(self, p: argparse.ArgumentParser) -> None:
+        p.add_argument(self.name, nargs=argparse.REMAINDER, help=self.help)
+
+
+@dataclass
+class Meta:
+    summary: str
+    args: List[object] = field(default_factory=list)
+
+
+def build_parser(meta: Meta, prog: str) -> argparse.ArgumentParser:
+    p = _Parser(prog=prog, description=meta.summary)
+    for a in meta.args:
+        a.add_to(p)
+    return p

scripts/engine/identity.py

@@ -0,0 +1,64 @@
+# scripts/engine/identity.py
+"""Node identity — fully explicit, from node.env. The framework assumes no domain
+and no clustering: a node declares its real FQDN and its public host name, and the
+short `name` is just the first label. The OS hostname is only a fallback for the FQDN.
+
+node.env keys:
+  APEX_NODE_FQDN  real FQDN (traefik Host() routing).            default: OS hostname
+  APEX_NODE_HOST  public host name (backup --host, obs/notify    default: APEX_NODE_FQDN
+                  `instance`); often the FQDN minus a cluster
+                  segment, but that's the node's choice.
+  APEX_SUBNET     the node's /24 for the `direct` network.       required
+"""
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass
+from typing import Dict, List, Tuple
+
+
+@dataclass
+class Node:
+    name: str    # short label (first segment of the host), e.g. "node1" — logs, branch, messages
+    subnet: str  # the node's direct-network /24
+    fqdn: str    # real FQDN — traefik Host() routing
+    host: str    # public host name — backup --host, obs `instance`, notify instance
+
+
+def read_env(path: str) -> Dict[str, str]:
+    """Parse a KEY=value file (shell-sourceable). Ignores comments/blank lines."""
+    out: Dict[str, str] = {}
+    if not os.path.isfile(path):
+        return out
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#") or "=" not in line:
+                continue
+            key, _, val = line.partition("=")
+            out[key.strip()] = val.strip().strip('"').strip("'")
+    return out
+
+
+def resolve(hostname: str, repo_root: str) -> Tuple[Node, List[str]]:
+    """Return (Node, warnings). Exits 66 (EX_NOINPUT) if APEX_SUBNET is missing."""
+    warns: List[str] = []
+    env = read_env(os.path.join(repo_root, "node.env"))
+
+    fqdn = env.get("APEX_NODE_FQDN", "") or (hostname or "").strip()
+    if not fqdn:
+        sys.stderr.write("identity: APEX_NODE_FQDN missing from node.env and no hostname\n")
+        raise SystemExit(66)
+    if not env.get("APEX_NODE_FQDN"):
+        warns.append(f"APEX_NODE_FQDN not set in node.env; using hostname {fqdn!r}.")
+
+    host = env.get("APEX_NODE_HOST", "") or fqdn      # node's public name; defaults to the FQDN
+    name = host.split(".")[0]                          # short label for logs/branch/messages
+
+    subnet = env.get("APEX_SUBNET", "")
+    if not subnet:
+        sys.stderr.write("identity: APEX_SUBNET missing from node.env\n")
+        raise SystemExit(66)
+
+    return Node(name=name, subnet=subnet, fqdn=fqdn, host=host), warns

scripts/engine/lib/crowdsec.py

@@ -0,0 +1,28 @@
+# scripts/engine/lib/crowdsec.py
+"""ctx.crowdsec — install + config deploy + collections (port of config/crowdsec)."""
+from __future__ import annotations
+import os
+import shutil
+
+
+class Crowdsec:
+    def __init__(self, log, sys_, tpl, host):
+        self.log, self.sys, self.tpl, self.host = log, sys_, tpl, host
+
+    def ensure_installed(self) -> None:
+        if shutil.which("cscli"):
+            return
+        self.log.info("Installing CrowdSec via official script...")
+        self.sys.run(["bash", "-c", "set -o pipefail; curl -sSfL https://install.crowdsec.net | sudo sh"],
+                     check=True)
+        self.sys.sudo(["apt-get", "install", "-y", "crowdsec"])
+
+    def deploy(self, configs_subdir: str, vars: dict, dry_run: bool = False) -> None:
+        for rel, content in self.tpl.render_dir(configs_subdir, vars).items():
+            self.host.write_file(os.path.join("/etc/crowdsec", rel), content,
+                                 backup=True, dry_run=dry_run, sudo=True)
+
+    def install_collections(self, collections) -> None:
+        self.sys.sudo(["cscli", "hub", "update"])
+        for c in collections:
+            self.sys.sudo(["cscli", "collections", "install", c])

scripts/engine/lib/host.py

@@ -0,0 +1,191 @@
+# scripts/engine/lib/host.py
+"""ctx.host — idempotent host mutation: backup, write_file, inject_block, install_crontab.
+
+Ports root_template_crontab (install + rollback) and the ufw block-injection logic
+(root_template.sh _root_template_inject_rules_to_ufw_file), generalized + self-healing.
+"""
+from __future__ import annotations
+
+import inspect
+import json
+import os
+import subprocess
+import time
+from typing import List, Optional, Sequence, Tuple
+
+
+def apply_block(text: str, content: str, start: str, end: str,
+                anchor: Optional[str] = None,
+                old_pairs: Sequence[Tuple[str, str]] = ()) -> Tuple[str, bool]:
+    """Pure transform: remove existing (start,end) and old_pairs blocks, insert a fresh
+    block after the first anchor line (or append if anchor is None). If an anchor is
+    given but not present, return (text, False) unchanged (matches bash skip behavior)."""
+    lines = text.splitlines()
+    if anchor is not None and not any(anchor in ln for ln in lines):
+        return text, False
+
+    pairs = [(start, end), *old_pairs]
+    stripped_starts = {s: e for s, e in pairs}
+    out: List[str] = []
+    skip_until: Optional[str] = None
+    for ln in lines:
+        if skip_until is None:
+            s = ln.strip()
+            if s in stripped_starts:
+                skip_until = stripped_starts[s]
+                continue
+            out.append(ln)
+        else:
+            if ln.strip() == skip_until:
+                skip_until = None
+            continue
+
+    block = [start, *content.splitlines(), end]
+    if anchor is None:
+        out.extend(block)
+        return "\n".join(out) + "\n", True
+
+    result: List[str] = []
+    inserted = False
+    for ln in out:
+        result.append(ln)
+        if not inserted and anchor in ln:
+            result.extend(block)
+            inserted = True
+    return "\n".join(result) + "\n", inserted
+
+
+def _atomic_write(path, content, mode):
+    """Self-contained so sudo executes the same operation without importing user code."""
+    import os
+    import stat
+    import tempfile
+
+    target = os.path.realpath(path)
+    parent = os.path.dirname(target)
+    try:
+        previous = os.stat(target)
+    except FileNotFoundError:
+        previous = None
+    if previous is not None and not stat.S_ISREG(previous.st_mode):
+        raise ValueError(f"Not a regular file: {path}")
+    target_mode = (int(mode, 8) if mode is not None else
+                   stat.S_IMODE(previous.st_mode) if previous is not None else 0o644)
+    os.makedirs(parent, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=parent)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="") as stream:
+            stream.write(content)
+            stream.flush()
+            if previous is not None:
+                current = os.fstat(stream.fileno())
+                if (current.st_uid, current.st_gid) != (previous.st_uid, previous.st_gid):
+                    os.fchown(stream.fileno(), previous.st_uid, previous.st_gid)
+            os.fchmod(stream.fileno(), target_mode)
+            os.fsync(stream.fileno())
+        os.replace(temporary, target)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def _atomic_backup(path, destination):
+    import os
+    import shutil
+    import tempfile
+
+    destination = os.path.abspath(destination)
+    fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=os.path.dirname(destination))
+    os.close(fd)
+    try:
+        shutil.copyfile(path, temporary)
+        previous, current = os.stat(path), os.stat(temporary)
+        if (current.st_uid, current.st_gid) != (previous.st_uid, previous.st_gid):
+            os.chown(temporary, previous.st_uid, previous.st_gid)
+        shutil.copystat(path, temporary)
+        os.replace(temporary, destination)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+class Host:
+    def __init__(self, log, sys_):
+        self.log = log
+        self.sys = sys_
+
+    def _file_operation(self, operation, args, sudo):
+        if sudo:
+            script = (inspect.getsource(operation) + "\nimport json, sys\n" +
+                      f"{operation.__name__}(*json.load(sys.stdin))\n")
+            self.sys.sudo(["python3", "-I", "-c", script], input=json.dumps(args))
+        else:
+            operation(*args)
+
+    # --- read/write with sudo awareness ---
+    def _read(self, path: str, sudo: bool = False) -> str:
+        if sudo:
+            cp = self.sys.sudo(["cat", path], check=True, capture=True)
+            return cp.stdout or ""
+        with open(path, encoding="utf-8") as f:
+            return f.read()
+
+    def backup(self, path: str, sudo: bool = False) -> Optional[str]:
+        exists = (self.sys.sudo(["test", "-f", path], check=False).returncode == 0 if sudo
+                  else os.path.isfile(path))
+        if not exists:
+            return None
+        bkp = f"{path}.{int(time.time())}.bak"
+        self._file_operation(_atomic_backup, [path, bkp], sudo)
+        self.log.info(f"Backed up {path} -> {bkp}")
+        return bkp
+
+    def write_file(self, path: str, content: str, mode: Optional[str] = None,
+                   backup: bool = True, dry_run: bool = False, sudo: bool = False) -> None:
+        if dry_run:
+            self.log.info(f"[DRY-RUN] would write {len(content)} bytes to {path}")
+            return
+        if backup:
+            self.backup(path, sudo=sudo)
+        self._file_operation(_atomic_write, [path, content, mode], sudo)
+        self.log.success(f"Wrote {path}")
+
+    def inject_block(self, target: str, content: str, start: str, end: str,
+                     anchor: Optional[str] = None,
+                     old_pairs: Sequence[Tuple[str, str]] = (),
+                     dry_run: bool = False, sudo: bool = True) -> bool:
+        exists = (self.sys.sudo(["test", "-f", target], check=False).returncode == 0 if sudo
+                  else os.path.isfile(target))
+        if not exists:
+            self.log.error(f"Target file does not exist: {target}")
+            return False
+        text = self._read(target, sudo=sudo)
+        new_text, injected = apply_block(text, content, start, end, anchor, old_pairs)
+        if not injected:
+            self.log.warn(f"Anchor {anchor!r} not found in {target}; skipping.")
+            return False
+        if dry_run:
+            self.log.info(f"[DRY-RUN] would inject block into {target}")
+            return True
+        self.write_file(target, new_text, backup=True, sudo=sudo)
+        self.log.success(f"Block injected into {target}")
+        return True
+
+    def install_crontab(self, text: str, dry_run: bool = False) -> None:
+        """Port of root_template_crontab: backup, install via `crontab -`, rollback."""
+        backup = subprocess.run(["crontab", "-l"], text=True,
+                                stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+        prev = backup.stdout if backup.returncode == 0 else ""
+        if not text.strip():
+            self.log.error("Rendered crontab is empty; aborting (previous kept).")
+            raise SystemExit(65)  # EX_DATAERR
+        self.log.info("Rendered crontab preview:\n" + text)
+        if dry_run:
+            self.log.info("[DRY-RUN] would install the crontab above.")
+            return
+        res = subprocess.run(["crontab", "-"], input=text, text=True)
+        if res.returncode != 0:
+            self.log.error("crontab install failed; restoring previous.")
+            subprocess.run(["crontab", "-"], input=prev, text=True)
+            raise SystemExit(1)
+        self.log.success("Crontab installed.")

scripts/engine/lib/log.py

@@ -0,0 +1,19 @@
+# scripts/engine/lib/log.py
+"""ctx.log — [node] [action] [level] message (port of root_log_*)."""
+from __future__ import annotations
+import sys
+
+
+class Log:
+    def __init__(self, node: str = "global", action: str = "core"):
+        self.node = node
+        self.action = action
+
+    def _emit(self, level: str, msg: str, stream=None) -> None:
+        print(f"[{self.node}] [{self.action}] [{level}] {msg}",
+              file=stream or sys.stdout, flush=True)
+
+    def info(self, msg): self._emit("INFO", msg)
+    def warn(self, msg): self._emit("WARN", msg)
+    def error(self, msg): self._emit("ERROR", msg, sys.stderr)
+    def success(self, msg): self._emit("SUCCESS", msg)

scripts/engine/lib/notify.py

@@ -0,0 +1,87 @@
+# scripts/engine/lib/notify.py
+"""ctx.notify — Telegram notifications.
+
+The alert `instance` is the node's public host (APEX_NODE_HOST), passed in from the
+context. POSTs application/x-www-form-urlencoded via urllib (stdlib).
+"""
+from __future__ import annotations
+
+import html
+import urllib.parse
+import urllib.request
+from datetime import datetime
+
+NOTIFY_TIMEOUT = 20
+MAX_MESSAGE_LEN = 3500
+
+
+def resolve_bot_url(value: str) -> str:
+    """Resolve a literal URL or an @file reference without putting secrets in argv."""
+    if not value or not value.startswith("@"):
+        return value
+    path = value[1:]
+    if not path:
+        raise ValueError("empty Telegram Bot URL file reference")
+    with open(path, encoding="utf-8") as f:
+        return f.read().strip()
+
+
+def build_text(title: str, message: str, node: str, level: str, ts: str) -> str:
+    status, icon, severity = "firing", "🚨", "info"
+    if level == "SUCCESS":
+        status, icon = "resolved", "✅"
+    elif level == "ERROR":
+        severity = "critical"
+    elif level == "WARN":
+        severity = "warning"
+    instance = node
+    msg = message if len(message) <= MAX_MESSAGE_LEN else message[:MAX_MESSAGE_LEN] + "..."
+    msg = html.escape(msg)
+    safe_title = html.escape(title)
+    return (f"{icon} <b>{safe_title}</b>\n"
+            f"<b>Instance:</b> {instance}\n"
+            f"<b>Timestamp:</b> {ts}\n"
+            f"<b>Status:</b> {status.capitalize()}\n"
+            f"<b>Escalation:</b> {severity}\n"
+            f"<pre><code>{msg}</code></pre>\n"
+            f"#status_{status} #escalation_{severity} #instance_{instance.replace('.', '_')}")
+
+
+class Notify:
+    def __init__(self, log, node: str = "global"):
+        self.log = log
+        self.node = node
+
+    def telegram(self, title, bot_url, message, node=None, level="INFO") -> bool:
+        node = node or self.node
+        try:
+            bot_url = resolve_bot_url(bot_url)
+        except (OSError, ValueError) as e:
+            self.log.error(f"Failed to read Telegram Bot URL: {e}")
+            return False
+        if not bot_url:
+            self.log.warn("Telegram Bot URL not provided. Skipping notification.")
+            return False
+
+        parsed = urllib.parse.urlparse(bot_url)
+        if parsed.scheme != "https":
+            self.log.error(f"Telegram Bot URL must use HTTPS, got scheme: {parsed.scheme!r}")
+            return False
+
+        ts = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
+        text = build_text(title, message, node, level, ts)
+        data = urllib.parse.urlencode({"parse_mode": "HTML", "text": text}).encode()
+        self.log.info(f"Sending Telegram notification ({level}): {title}")
+        try:
+            with urllib.request.urlopen(urllib.request.Request(bot_url, data=data),  # noqa: S310 -- HTTPS validated above.
+                                        timeout=NOTIFY_TIMEOUT) as r:
+                r.read()
+            return True
+        except Exception as e:
+            self.log.error(f"Failed to send Telegram notification: {e}")
+            return False
+
+    def success(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "SUCCESS")
+    def error(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "ERROR")
+    def info(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "INFO")
+    def warn(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "WARN")

scripts/engine/lib/sys.py

@@ -0,0 +1,98 @@
+# scripts/engine/lib/sys.py
+"""ctx.sys — command execution + systemd service management (port of root_sys.sh)."""
+from __future__ import annotations
+import os
+import shutil
+import subprocess
+import time
+from typing import List, Optional, Sequence
+
+
+class Sys:
+    def __init__(self, log):
+        self.log = log
+
+    # --- command execution ---
+    def run(self, cmd: Sequence[str], check: bool = True, capture: bool = False,
+            cwd: Optional[str] = None, env: Optional[dict] = None,
+            input: Optional[str] = None) -> subprocess.CompletedProcess:
+        return subprocess.run(list(cmd), check=check, cwd=cwd, env=env, text=True,
+                              input=input,
+                              stdout=subprocess.PIPE if capture else None,
+                              stderr=subprocess.PIPE if capture else None)
+
+    def sudo(self, cmd: Sequence[str], **kw) -> subprocess.CompletedProcess:
+        if os.geteuid() == 0 and shutil.which("sudo") is None:
+            return self.run(cmd, **kw)
+        return self.run(["sudo", *cmd], **kw)
+
+    def ok(self, cmd: Sequence[str], **kw) -> bool:
+        """Run, return True on exit 0, never raising."""
+        return self.run(cmd, check=False, **kw).returncode == 0
+
+    # --- systemd (port of root_sys_*) ---
+    def service_exists(self, name: str) -> bool:
+        cp = self.run(["systemctl", "list-unit-files", "--type=service"],
+                      check=False, capture=True)
+        return any(line.startswith(f"{name}.service")
+                   for line in (cp.stdout or "").splitlines())
+
+    def service_is_active(self, name: str) -> bool:
+        return self.ok(["systemctl", "is-active", "--quiet", name])
+
+    def service_is_enabled(self, name: str) -> bool:
+        return self.ok(["systemctl", "is-enabled", "--quiet", name])
+
+    def start_and_enable(self, name: str) -> bool:
+        if not self.service_exists(name):
+            self.log.warn(f"Service {name} does not exist on this system")
+            return False
+        if not self.service_is_active(name):
+            self.log.info(f"Starting {name} service...")
+            self.sudo(["systemctl", "start", name])
+        if not self.service_is_enabled(name):
+            self.log.info(f"Enabling {name} service...")
+            self.sudo(["systemctl", "enable", name])
+        return self.service_is_active(name)
+
+    def ensure_running(self, name: str) -> None:
+        """Port of root_init_ensure_service_running: enable --now if not active."""
+        if not self.service_is_active(name):
+            self.log.info(f"Starting {name} service...")
+            self.sudo(["systemctl", "enable", "--now", name])
+        else:
+            self.log.info(f"{name} service already running.")
+
+    def restart(self, name: str) -> bool:
+        self.log.info(f"Restarting {name} service...")
+        self.sudo(["systemctl", "restart", name])
+        return self.service_is_active(name)
+
+    def reload(self, name: str, dry_run: bool = False) -> None:
+        if dry_run:
+            self.log.info(f"[DRY-RUN] systemctl reload {name}")
+            return
+        self.log.info(f"Reloading {name} configuration...")
+        self.sudo(["systemctl", "reload", name])
+
+    def daemon_reload(self) -> None:
+        self.sudo(["systemctl", "daemon-reload"])
+
+    def wait_for(self, name: str, timeout: int = 30, interval: int = 2) -> bool:
+        elapsed = 0
+        while elapsed < timeout:
+            if self.service_is_active(name):
+                return True
+            time.sleep(interval)
+            elapsed += interval
+        return False
+
+    def install_packages(self, packages: List[str]) -> None:
+        """Port of root_init_install_packages (install if the binary is absent)."""
+        for pkg in packages:
+            if shutil.which(pkg):
+                self.log.info(f"{pkg} already installed.")
+                continue
+            self.log.info(f"Installing {pkg}...")
+            self.sudo(["apt-get", "update", "-y"])
+            self.sudo(["apt-get", "install", "-y", pkg])

scripts/engine/lib/systemd.py

@@ -0,0 +1,26 @@
+# scripts/engine/lib/systemd.py
+"""ctx.systemd — render + install unit files, daemon-reload, enable+start."""
+from __future__ import annotations
+import os
+
+
+class Systemd:
+    def __init__(self, log, sys_, tpl, host):
+        self.log, self.sys, self.tpl, self.host = log, sys_, tpl, host
+
+    def deploy_units(self, configs_subdir: str, vars: dict, dry_run: bool = False) -> None:
+        # Top-level, non-hidden files only — the bash glob ("$CONFIG_DIR"/* + [[ -f ]])
+        # never recursed nor picked up dotfiles like .gitkeep.
+        rendered = {rel: c for rel, c in self.tpl.render_dir(configs_subdir, vars).items()
+                    if "/" not in rel and not rel.startswith(".")}
+        if not rendered:
+            self.log.warn(f"No systemd units found under configs/{configs_subdir}.")
+            return
+        for rel, content in sorted(rendered.items()):
+            name = os.path.basename(rel)
+            self.host.write_file(f"/etc/systemd/system/{name}", content,
+                                 backup=True, dry_run=dry_run, sudo=True)
+            if name.endswith(".service") and not dry_run:
+                self.sys.daemon_reload()
+                self.sys.ensure_running(name[: -len(".service")])
+                self.log.success(f"Service {name[:-8]} active and enabled.")

scripts/engine/lib/tpl.py

@@ -0,0 +1,37 @@
+# scripts/engine/lib/tpl.py
+"""ctx.tpl — stdlib string.Template rendering.
+
+safe_substitute replaces known $VAR/${VAR} and leaves unknown ones literal.
+Shell $(...) command-substitution survives untouched.
+Loops/conditionals live in the python action, never in the template file.
+"""
+from __future__ import annotations
+import os
+from string import Template
+from typing import Dict
+
+
+class Tpl:
+    def __init__(self, configs_dir: str, log):
+        self.configs_dir = configs_dir
+        self.log = log
+
+    def _resolve(self, path: str) -> str:
+        return path if os.path.isabs(path) else os.path.join(self.configs_dir, path)
+
+    def render(self, path: str, vars: Dict[str, str]) -> str:
+        full = self._resolve(path)
+        with open(full) as f:
+            return Template(f.read()).safe_substitute(vars)
+
+    def render_dir(self, rel_dir: str, vars: Dict[str, str]) -> Dict[str, str]:
+        """Render every file under configs/<rel_dir>. Returns {relpath: content}."""
+        base = self._resolve(rel_dir)
+        out: Dict[str, str] = {}
+        for dirpath, _dirs, files in os.walk(base):
+            for fn in sorted(files):
+                full = os.path.join(dirpath, fn)
+                rel = os.path.relpath(full, base).replace(os.sep, "/")
+                with open(full) as f:
+                    out[rel] = Template(f.read()).safe_substitute(vars)
+        return out

scripts/engine/lib/ufw.py

@@ -0,0 +1,105 @@
+# scripts/engine/lib/ufw.py
+"""Apply ufw and ufw-docker rules.
+
+Replace existing supported marker blocks before inserting current rules.
+Docker rules go through ufw-docker; host rules through plain ufw.
+"""
+from __future__ import annotations
+
+import os
+import shlex
+import shutil
+
+APEX_START = "# START APEX.ERMNVLDMR.COM RULES"
+APEX_END = "# END APEX.ERMNVLDMR.COM RULES"
+OLD_PAIRS = [("# START ROOT.ERMNVLDMR.COM RULES", "# END ROOT.ERMNVLDMR.COM RULES")]
+
+
+def resolve_ufw_docker_bin() -> str:
+    """Find ufw-docker in PATH, /usr/local/bin, or fallback to ~/.local/bin/ufw-docker."""
+    found = shutil.which("ufw-docker")
+    if found:
+        return found
+    if os.path.isfile("/usr/local/bin/ufw-docker") and os.access("/usr/local/bin/ufw-docker", os.X_OK):
+        return "/usr/local/bin/ufw-docker"
+    return os.path.expanduser("~/.local/bin/ufw-docker")
+
+
+UFW_DOCKER_BIN = resolve_ufw_docker_bin()
+
+
+def clean_rules(text: str):
+    """Strip whitespace, drop blank + comment lines (matches the bash awk filter)."""
+    return [s for s in (ln.strip() for ln in text.splitlines()) if s and not s.startswith("#")]
+
+
+class Ufw:
+    def __init__(self, log, sys_, host):
+        self.log, self.sys, self.host = log, sys_, host
+
+    def _inject_file(self, target: str, source_file: str, dry_run: bool):
+        if not os.path.isfile(source_file):
+            return
+        if not self.sys.ok(["sudo", "test", "-f", target]):
+            self.log.error(f"Target file does not exist: {target}")
+            raise SystemExit(1)
+        with open(source_file, encoding="utf-8") as f:
+            content = "\n".join(clean_rules(f.read()))
+        self.host.inject_block(
+            target,
+            content,
+            APEX_START,
+            APEX_END,
+            anchor="# End required lines",
+            old_pairs=OLD_PAIRS,
+            dry_run=dry_run,
+            sudo=True,
+        )
+
+    def apply(self, rules_dir: str | None = None, config_dir: str | None = None, dry_run: bool = False) -> None:
+        if rules_dir and not os.path.isdir(rules_dir):
+            self.log.error(f"Rules directory does not exist: {rules_dir}")
+            raise SystemExit(66)
+        if config_dir and not os.path.isdir(config_dir):
+            self.log.error(f"Config directory does not exist: {config_dir}")
+            raise SystemExit(66)
+        if not rules_dir and not config_dir:
+            self.log.warn("Neither docker nor host ufw rules directory found; nothing to apply.")
+            return
+        if not self.sys.ok(["sudo", "ufw", "--version"]):
+            self.log.error("ufw not installed/working via sudo. Run config/packages first.")
+            raise SystemExit(1)
+        if not (os.path.isfile(UFW_DOCKER_BIN) and os.access(UFW_DOCKER_BIN, os.X_OK)):
+            self.log.error(f"ufw-docker not found at {UFW_DOCKER_BIN}. Run config/ufw-docker first.")
+            raise SystemExit(1)
+
+        if config_dir:
+            host_rules = os.path.join(config_dir, "host.rules")
+            if os.path.isfile(host_rules):
+                with open(host_rules, encoding="utf-8") as f:
+                    rules = clean_rules(f.read())
+                for rule in rules:
+                    self.log.info(f"Applying host ufw rule: {rule}")
+                    if dry_run:
+                        self.log.info(f"[DRY-RUN] sudo ufw {rule}")
+                    else:
+                        self.sys.sudo(["ufw", *shlex.split(rule)], check=True)
+            self._inject_file("/etc/ufw/before.rules", os.path.join(config_dir, "before.rules"), dry_run)
+            self._inject_file("/etc/ufw/after.rules", os.path.join(config_dir, "after.rules"), dry_run)
+
+        if rules_dir:
+            rule_files = [fn for fn in sorted(os.listdir(rules_dir)) if fn.endswith(".rules") and not fn.startswith(".")]
+            if not rule_files:
+                self.log.warn(f"No .rules files found in {rules_dir}, nothing to apply.")
+            for fn in rule_files:
+                with open(os.path.join(rules_dir, fn), encoding="utf-8") as f:
+                    rules = clean_rules(f.read())
+                for rule in rules:
+                    self.log.info(f"Applying ufw-docker rule ({fn}): {rule}")
+                    if dry_run:
+                        self.log.info(f"[DRY-RUN] sudo {UFW_DOCKER_BIN} {rule}")
+                    else:
+                        self.sys.sudo([UFW_DOCKER_BIN, *shlex.split(rule)], check=True)
+
+        self.log.success("All ufw rules applied.")
+        self.log.warn("Run `sudo systemctl restart ufw` to apply. Ensure ssh is allowed via plain ufw!")

scripts/engine/overlay.py

@@ -0,0 +1,116 @@
+# scripts/engine/overlay.py
+"""Resolve action names across commons + proprietaries.
+
+proprietaries first (override), commons second. Listing uses static ast scanning
+(no import side effects). Dispatch loads exactly one module by path.
+"""
+from __future__ import annotations
+
+import ast
+import importlib.util
+import os
+import re
+from dataclasses import dataclass
+from typing import List, Optional
+
+ACTION_EXT = ".py"
+ACTION_NAME = re.compile(r"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+")
+
+
+@dataclass
+class Resolved:
+    name: str
+    path: str
+    shadowed: Optional[str]   # commons path when a proprietaries action overrides it
+
+
+@dataclass
+class ActionInfo:
+    name: str
+    source: str               # "commons" | "local" | "override"
+    summary: str
+    disabled: object          # None | True | "reason"
+
+
+def _rel_names(root: Optional[str]):
+    if not root or not os.path.isdir(root):
+        return {}
+    out = {}
+    for dirpath, _dirs, files in os.walk(root):
+        for fn in files:
+            if not fn.endswith(ACTION_EXT):
+                continue
+            full = os.path.join(dirpath, fn)
+            rel = os.path.relpath(full, root)[: -len(ACTION_EXT)]
+            name = rel.replace(os.sep, "/")
+            if ACTION_NAME.fullmatch(name):
+                out[name] = full
+    return out
+
+
+def resolve(name: str, commons_root: str, prop_root: Optional[str]) -> Optional[Resolved]:
+    if not ACTION_NAME.fullmatch(name):
+        return None
+    rel = name + ACTION_EXT
+    commons_path = os.path.join(commons_root, rel)
+    c = os.path.isfile(commons_path)
+    prop_path = os.path.join(prop_root, rel) if prop_root else None
+    p = bool(prop_path) and os.path.isfile(prop_path)
+    if p and c:
+        return Resolved(name, prop_path, commons_path)
+    if p:
+        return Resolved(name, prop_path, None)
+    if c:
+        return Resolved(name, commons_path, None)
+    return None
+
+
+def discover(commons_root: str, prop_root: Optional[str]) -> List[ActionInfo]:
+    commons = _rel_names(commons_root)
+    prop = _rel_names(prop_root)
+    infos = []
+    for name in sorted(set(commons) | set(prop)):
+        if name in prop and name in commons:
+            source, path = "override", prop[name]
+        elif name in prop:
+            source, path = "local", prop[name]
+        else:
+            source, path = "commons", commons[name]
+        summary, disabled = _scan(path)
+        infos.append(ActionInfo(name, source, summary, disabled))
+    return infos
+
+
+def _scan(path: str):
+    """Statically extract (summary, DISABLED) without importing the module."""
+    try:
+        with open(path) as f:
+            tree = ast.parse(f.read(), path)
+    except (SyntaxError, OSError):
+        return "(unparseable)", None
+    summary, disabled = "", None
+    for node in tree.body:
+        if not isinstance(node, ast.Assign):
+            continue
+        targets = [t.id for t in node.targets if isinstance(t, ast.Name)]
+        if "METADATA" in targets and isinstance(node.value, ast.Call):
+            summary = _summary_of(node.value)
+        if "DISABLED" in targets and isinstance(node.value, ast.Constant):
+            disabled = node.value.value if node.value.value else None
+    return summary, disabled
+
+
+def _summary_of(call: ast.Call) -> str:
+    for kw in call.keywords:
+        if kw.arg == "summary" and isinstance(kw.value, ast.Constant):
+            return kw.value.value
+    if call.args and isinstance(call.args[0], ast.Constant):
+        return call.args[0].value
+    return ""
+
+
+def load_module(path: str, modname: str):
+    spec = importlib.util.spec_from_file_location(modname, path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod

scripts/githooks/pre-commit

@@ -0,0 +1,9 @@
+#!/bin/bash
+# apex commons pre-commit: lint staged docker-compose files.
+if [[ -n "${APEX_COMMONS:-}" ]]; then
+    APEX_CMD="$APEX_COMMONS/apex"
+else
+    APEX_CMD="$(git rev-parse --show-toplevel)/commons/apex"
+    [[ -x "$APEX_CMD" ]] || APEX_CMD="$(git rev-parse --show-toplevel)/apex"  # standalone commons repo
+fi
+"$APEX_CMD" util/lint-docker-compose --hook || { echo "[pre-commit] Aborting: apex util/lint-docker-compose failed."; exit 1; }

scripts/provisioning/__init__.py

@@ -0,0 +1 @@
+"""Host-side Docker installation and SSH handover."""

scripts/provisioning/access.py

@@ -0,0 +1,597 @@
+"""Protected SSH handover with a separately locked, persistent rollback timer."""
+from __future__ import annotations
+
+import argparse
+import base64
+import fcntl
+import fnmatch
+import hashlib
+import ipaddress
+import json
+import os
+from pathlib import Path
+import pwd
+import re
+import shlex
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+
+
+class AccessError(RuntimeError):
+    pass
+
+
+def include_arguments(value):
+    arguments, word, quote, started = [], [], None, False
+    for character in value:
+        if character == '\\':
+            raise AccessError('Escaped SSH Include paths require manual reconciliation')
+        if quote:
+            if character == quote:
+                quote = None
+            else:
+                word.append(character)
+        elif character in ('"', "'"):
+            quote, started = character, True
+        elif character in ' \t\r':
+            if started:
+                arguments.append(''.join(word))
+                word, started = [], False
+        elif character == '#' and not started:
+            break
+        else:
+            word.append(character)
+            started = True
+    if quote:
+        raise AccessError('Unclosed SSH Include quote requires manual reconciliation')
+    if started:
+        arguments.append(''.join(word))
+    if not arguments or len(arguments) > 128 or any(not item or len(item) > 4096 for item in arguments):
+        raise AccessError('Unsupported SSH Include paths require manual reconciliation')
+    return arguments
+
+
+MANUAL_PROFILES = ('debian', 'proxmox')
+OPERATOR_MATCH = """Match User adam
+AuthenticationMethods publickey
+PubkeyAuthentication yes
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+PermitEmptyPasswords no
+X11Forwarding no""".splitlines()
+
+
+class Host:
+    def __init__(self, root=Path('/'), runner=subprocess.run, profile=None):
+        if profile is not None and profile not in MANUAL_PROFILES:
+            raise AccessError('Unknown SSH access profile')
+        self.profile = profile
+        self.root = Path(root).resolve()
+        self.runner = runner
+
+    def path(self, path):
+        return self.root / path.lstrip('/')
+
+    def command(self, args, check=True):
+        result = self.runner(args, capture_output=True, text=True, timeout=60)
+        if check and result.returncode:
+            raise AccessError('Access command failed: ' + args[0])
+        return result
+
+    def boot_id(self):
+        return self.path('/proc/sys/kernel/random/boot_id').read_text().strip()
+
+    def fingerprint(self):
+        digest = hashlib.sha256()
+        paths = [self.path('/etc/ssh/sshd_config'), self.path('/etc/default/ssh')]
+        for directory in ('/etc/ssh/sshd_config.d', '/etc/systemd/system/ssh.service.d',
+                          '/run/systemd/system/ssh.service.d', '/usr/lib/systemd/system/ssh.service.d',
+                          '/lib/systemd/system/ssh.service.d'):
+            paths += sorted(self.path(directory).glob('*.conf'))
+        for path in paths:
+            digest.update(str(path.relative_to(self.root)).encode())
+            digest.update(path.read_bytes() if path.exists() else b'<absent>')
+        return digest.hexdigest()
+
+    def unconditional_configuration(self):
+        remaining, directory_entries, visited = 1024 * 1024, 0, set()
+
+        def expand(pattern):
+            nonlocal directory_entries
+            if any(token in pattern for token in ('[:', '[.', '[=', '[^')):
+                raise AccessError('Extended SSH Include glob syntax requires manual reconciliation')
+            absolute = pattern if pattern.startswith('/') else '/etc/ssh/' + pattern
+            mapped = self.path(absolute)
+            if len(mapped.parts) > 64:
+                raise AccessError('SSH Include path exceeds supported limits; manual reconciliation required')
+            candidates = [Path(mapped.anchor)]
+            for component in mapped.parts[1:]:
+                expanded = []
+                for parent in candidates:
+                    if not any(character in component for character in '*?['):
+                        candidate = parent / component
+                        if os.path.lexists(candidate):
+                            expanded.append(candidate)
+                        continue
+                    try:
+                        with os.scandir(parent) as entries:
+                            for entry in entries:
+                                directory_entries += 1
+                                if directory_entries > 4096:
+                                    raise AccessError('SSH Include traversal exceeds supported limits; manual reconciliation required')
+                                if entry.name.startswith('.') and not component.startswith('.'):
+                                    continue
+                                if fnmatch.fnmatchcase(entry.name, component):
+                                    expanded.append(Path(entry.path))
+                    except (FileNotFoundError, NotADirectoryError):
+                        continue
+                if len(expanded) > 128:
+                    raise AccessError('SSH Include expansion exceeds supported limits; manual reconciliation required')
+                candidates = expanded
+            return sorted(candidates)
+
+        def inspect(path, ancestors):
+            nonlocal remaining
+            information = path.stat()
+            identity = (information.st_dev, information.st_ino)
+            if identity in ancestors or len(ancestors) >= 16:
+                raise AccessError('Recursive SSH Include configuration requires manual reconciliation')
+            if identity in visited:
+                return
+            if len(visited) >= 128 or not stat.S_ISREG(information.st_mode) or information.st_size > remaining:
+                raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required')
+            visited.add(identity)
+            with path.open('rb') as stream:
+                content = stream.read(remaining + 1)
+            remaining -= len(content)
+            if remaining < 0:
+                raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required')
+            lines = content.decode().splitlines()
+            if self.profile and path == self.path('/etc/ssh/sshd_config'):
+                # The legacy action appended this exact terminal block. Any additional
+                # directives would change its meaning and require manual inspection.
+                marker = '# APEX operator SSH policy'
+                if marker in lines:
+                    position = lines.index(marker)
+                    if [item.strip() for item in lines[position + 1:] if item.strip()] == OPERATOR_MATCH:
+                        lines = lines[:position]
+            for line in lines:
+                line = line.lstrip()
+                if not line or line.startswith('#'):
+                    continue
+                if line.startswith('"'):
+                    keyword, separator, value = line[1:].partition('"')
+                    if not separator:
+                        raise AccessError('Unsupported SSH keyword quoting requires manual reconciliation')
+                else:
+                    parsed = re.match(r'([^\s=]+)(.*)', line)
+                    if not parsed:
+                        raise AccessError('Unsupported SSH configuration syntax requires manual reconciliation')
+                    keyword, value = parsed.groups()
+                    value = value.lstrip()
+                    if value.startswith('='):
+                        value = value[1:]
+                if keyword.lower() == 'match':
+                    raise AccessError('SSH Match directives require manual reconciliation before initialization')
+                if keyword.lower() != 'include':
+                    continue
+                for pattern in include_arguments(value):
+                    for included in expand(pattern):
+                        inspect(included, ancestors | {identity})
+
+        try:
+            inspect(self.path('/etc/ssh/sshd_config'), set())
+        except (OSError, UnicodeError) as exc:
+            raise AccessError('Cannot inspect SSH configuration and includes; manual reconciliation required') from exc
+
+    def bootstrap_preflight(self):
+        for activity in ('is-active', 'is-enabled'):
+            if self.command(['systemctl', activity, '--quiet', 'ssh.socket'], False).returncode == 0:
+                raise AccessError('SSH socket activation requires manual handover; no access changes made')
+        if self.command(['systemctl', 'is-active', '--quiet', 'ssh.service'], False).returncode:
+            raise AccessError('An active ssh.service is required')
+        mode = self.command(['systemctl', 'show', 'ssh.service', '-p', 'KillMode', '--value']).stdout.strip()
+        if mode != 'process':
+            raise AccessError('ssh.service must preserve established sessions with KillMode=process')
+        arguments = self.service_arguments()
+        managed = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        if managed.exists():
+            commands = [line.removeprefix('ExecStart=') for line in managed.read_text().splitlines()
+                        if line.startswith('ExecStart=') and line != 'ExecStart=']
+            if len(commands) != 1 or arguments != shlex.split(commands[0]):
+                raise AccessError('Active SSH service configuration differs from its managed invocation')
+        elif arguments != ['/usr/sbin/sshd', '-D']:
+            raise AccessError('Custom SSH service invocation requires manual access reconciliation')
+        defaults = self.path('/etc/default/ssh')
+        if defaults.exists():
+            for line in defaults.read_text().splitlines():
+                if re.match(r'^\s*SSHD_OPTS\s*=', line):
+                    value = line.split('=', 1)[1].strip().strip('\"\'')
+                    if value:
+                        raise AccessError('Custom SSHD_OPTS require manual access reconciliation')
+        self.unconditional_configuration()
+        self.command(['/usr/sbin/sshd', '-t'])
+        self.live_arguments()
+
+    def preflight(self):
+        self.bootstrap_preflight()
+        self.command(['visudo', '-c'])
+        self.command(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true'])
+
+    def service_arguments(self):
+        output = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout
+        matches = re.findall(r'argv\[\]=([^;]+)', output)
+        if len(matches) != 1:
+            raise AccessError('Unable to verify the SSH service invocation')
+        arguments = shlex.split(matches[0])
+        # Debian's stock unit has this optional variable. Its actual expansion is
+        # independently checked against /proc below, so a nonempty value fails.
+        if arguments == ['/usr/sbin/sshd', '-D', '$SSHD_OPTS']:
+            arguments.pop()
+        if not arguments or arguments[0] != '/usr/sbin/sshd':
+            raise AccessError('SSH service does not invoke the supported daemon')
+        return arguments
+
+    def live_arguments(self):
+        expected = self.service_arguments()
+        pid = self.command(['systemctl', 'show', 'ssh.service', '-p', 'MainPID', '--value']).stdout.strip()
+        if not pid.isdecimal() or int(pid) <= 1:
+            raise AccessError('SSH service has no verifiable active listener process')
+        process = self.path('/proc/' + pid)
+        try:
+            if os.readlink(process / 'exe') != '/usr/sbin/sshd':
+                raise AccessError('Active SSH process does not run the supported daemon')
+            parts = [part for part in (process / 'cmdline').read_bytes().decode().split('\0') if part]
+        except (OSError, UnicodeError):
+            raise AccessError('Unable to inspect the active SSH listener invocation') from None
+        if len(parts) == 1 and parts[0].startswith('sshd: ') and ' [listener]' in parts[0]:
+            actual = shlex.split(parts[0][6:].split(' [listener]', 1)[0])
+        else:
+            actual = parts
+        if actual != expected:
+            raise AccessError('Running SSH listener differs from the active service invocation')
+        return actual, pid
+
+    def verify_active(self, ports, final, address):
+        args, pid = self.live_arguments()
+        self.verify_policy([arg for arg in args if arg != '-D'], ports, final, address)
+        if self.listener_ports(pid) != set(ports):
+            raise AccessError('Active SSH listening ports differ from the declared policy')
+
+    def listener_ports(self, pid):
+        output = self.command(['ss', '-H', '-ltnp']).stdout
+        actual_ports = set()
+        for line in output.splitlines():
+            if '("sshd",' not in line:
+                continue
+            if set(re.findall(r'pid=(\d+)', line)) != {pid}:
+                raise AccessError('An additional SSH listener exists outside the managed service')
+            fields = line.split()
+            if len(fields) < 5 or not fields[3].rsplit(':', 1)[-1].isdecimal():
+                raise AccessError('Unable to verify the active SSH listening sockets')
+            actual_ports.add(int(fields[3].rsplit(':', 1)[-1]))
+        return actual_ports
+
+    @property
+    def final_ports(self):
+        return [22, 2222] if self.profile == 'proxmox' else [2222]
+
+    def authentication_options(self):
+        proxmox = self.profile == 'proxmox'
+        options = {'PermitRootLogin': 'prohibit-password' if proxmox else 'no',
+                   'PasswordAuthentication': 'no', 'KbdInteractiveAuthentication': 'no',
+                   'PubkeyAuthentication': 'yes', 'AuthenticationMethods': 'publickey'}
+        if not proxmox:
+            options['AllowUsers'] = 'adam'
+        options.update(X11Forwarding='no', PermitEmptyPasswords='no')
+        return options
+
+    def policy(self, ports, final, address):
+        args = ['/usr/sbin/sshd']
+        for port in sorted(set(ports)):
+            args += ['-p', str(port)]
+        if final:
+            for key, value in self.authentication_options().items():
+                args += ['-o', f'{key}={value}']
+        elif self.profile:
+            # Keep existing managed authentication options when adding the probe port.
+            current = iter(self.service_arguments()[1:])
+            for flag in current:
+                if flag == '-D':
+                    continue
+                value = next(current, '')
+                if flag == '-p' and value.isdecimal():
+                    continue
+                if flag != '-o' or not re.fullmatch(r'[A-Za-z]+=[A-Za-z0-9,-]+', value):
+                    raise AccessError('Cannot safely extend the existing SSH invocation')
+                args += [flag, value]
+        self.verify_policy(args, ports, final, address)
+        return args
+
+    def verify_policy(self, args, ports, final, address):
+        self.unconditional_configuration()
+        self.command(args + ['-t'])
+        users = {'adam', 'root'}
+        if final:
+            users.update(account.pw_name for account in pwd.getpwall())
+        for user in sorted(users):
+            output = self.command(args + ['-T', '-C', f'user={user},host=localhost,addr={address}']).stdout
+            values = {}
+            for line in output.splitlines():
+                key, _, value = line.partition(' ')
+                values.setdefault(key, []).append(value)
+            if set(values.get('port', [])) != {str(p) for p in ports}:
+                raise AccessError('Effective SSH ports do not match the handover policy')
+            if final:
+                expected = {key.lower(): [value] for key, value in self.authentication_options().items()}
+                if self.profile == 'proxmox':
+                    if values.get('permitrootlogin') == ['without-password']:
+                        values['permitrootlogin'] = ['prohibit-password']
+                    if any(key in values for key in ('allowusers', 'denyusers', 'allowgroups', 'denygroups')):
+                        raise AccessError('SSH account restrictions conflict with Proxmox root-key access')
+                if any(values.get(key) != value for key, value in expected.items()):
+                    raise AccessError('Effective SSH authentication policy is inconsistent')
+
+    def install_policy(self, ports, final, address):
+        args = self.policy(ports, final, address)
+        path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        atomic(path, ('[Service]\nExecStart=\nExecStart=' + ' '.join(args + ['-D']) + '\n').encode(), 0o644)
+        self.command(['systemctl', 'daemon-reload'])
+        actual = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout
+        match = re.search(r'argv\[\]=([^;]+)', actual)
+        if not match or shlex.split(match[1]) != args + ['-D']:
+            raise AccessError('Another service override prevents the managed SSH policy')
+        self.command(['systemctl', 'restart', 'ssh.service'])
+        self.verify_active(ports, final, address)
+
+    def arm(self):
+        bundle = Path(__file__).resolve().parents[1]
+        if not re.fullmatch(r'/usr/local/lib/apex-provisioner/[a-f0-9]{64}', str(bundle)):
+            raise AccessError('Access runner must execute from the installed root-owned bundle')
+        service = ('[Unit]\nDescription=APEX pending access rollback\nAfter=local-fs.target\n'
+                   '[Service]\nType=oneshot\nWorkingDirectory=' + str(bundle) + '\n'
+                   'ExecStart=/usr/bin/python3 -m provisioning.access tick\n')
+        timer = ('[Unit]\nDescription=APEX access rollback deadline and boot recovery\n'
+                 '[Timer]\nOnBootSec=5s\nOnUnitActiveSec=10s\nAccuracySec=1s\n'
+                 '[Install]\nWantedBy=timers.target\n')
+        atomic(self.path('/etc/systemd/system/apex-access-rollback.service'), service.encode(), 0o644)
+        atomic(self.path('/etc/systemd/system/apex-access-rollback.timer'), timer.encode(), 0o644)
+        self.command(['systemctl', 'daemon-reload'])
+        self.command(['systemctl', 'enable', '--now', 'apex-access-rollback.timer'])
+
+    def disarm(self):
+        self.command(['systemctl', 'disable', '--now', 'apex-access-rollback.timer'])
+
+    def restore(self, state):
+        path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        if state['previous'] is None:
+            path.unlink(missing_ok=True)
+        else:
+            atomic(path, base64.b64decode(state['previous']), 0o644)
+        self.command(['/usr/sbin/sshd', '-t'])
+        self.command(['systemctl', 'daemon-reload'])
+        self.command(['systemctl', 'restart', 'ssh.service'])
+
+
+def atomic(path, data, mode=0o600):
+    path = Path(path)
+    for parent in [path, *path.parents]:
+        if parent.is_symlink():
+            raise AccessError('Refusing a symlink in a managed access path')
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix='.apex-access-', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'wb') as stream:
+            os.fchmod(stream.fileno(), mode)
+            stream.write(data)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+    directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(directory)
+    finally:
+        os.close(directory)
+
+
+class Transition:
+    def __init__(self, host=None, now=time.monotonic, profile=None):
+        if profile is not None and profile not in MANUAL_PROFILES:
+            raise AccessError('Unknown SSH access profile')
+        self.profile = profile
+        self.host = host or Host()
+        self.host.profile = profile
+        self.now = now
+        self.directory = self.host.path('/var/lib/apex-init/access')
+
+    def load(self):
+        path = self.directory / 'manual-state.json'
+        return json.loads(path.read_text()) if path.exists() else None
+
+    def save(self, state):
+        atomic(self.directory / 'manual-state.json', json.dumps(state, sort_keys=True).encode())
+
+    def execute(self, operation, token=None, original_port=None, client_address=None):
+        # The watchdog must recover SSH even when another APEX action holds init.lock.
+        if operation in ('tick', 'rollback'):
+            return self._execute(operation, token, original_port, client_address)
+        from .state import ProvisionError, exclusive_lock
+        try:
+            with exclusive_lock(self.host.path('/var/lib/apex-init/init.lock')):
+                return self._execute(operation, token, original_port, client_address)
+        except ProvisionError as error:
+            raise AccessError(str(error)) from None
+
+    def _execute(self, operation, token=None, original_port=None, client_address=None):
+        if token is not None and not re.fullmatch(r'[a-f0-9]{32}', token):
+            raise AccessError('Invalid access transition token')
+        for path in [self.directory, *self.directory.parents]:
+            if path.is_symlink():
+                raise AccessError('Invalid access state directory')
+        self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+        if self.directory.stat().st_uid != os.geteuid():
+            raise AccessError('Access state directory is not owned by the provisioner user')
+        os.chmod(self.directory, 0o700)
+        descriptor = os.open(self.directory / 'lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+        with os.fdopen(descriptor, 'w') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            state = self.load()
+            active = ('transition', 'final', 'recovery_pending')
+            # A historical bundle owns its pending rollback and the shared timer.
+            legacy_path = self.directory / 'state.json'
+            if legacy_path.exists():
+                legacy = json.loads(legacy_path.read_text())
+                if legacy['status'] in active or legacy.get('disarm_pending'):
+                    raise AccessError('Another access workflow is pending; wait for rollback before retrying')
+            if state and self.profile and state.get('profile') != self.profile and operation != 'tick':
+                if operation != 'begin' or state['status'] in active:
+                    raise AccessError('Access transition belongs to a different platform')
+                state = None
+            if state and state.get('disarm_pending') and state['status'] in ('committed', 'rolled_back'):
+                self._disarm(state)
+            if (state and token and token == state.get('token')
+                    and (operation, state['status']) in (('commit', 'committed'), ('rollback', 'rolled_back'))):
+                return {'status': state['status']}
+            if state and state['status'] == 'recovery_pending':
+                self._rollback(state)
+                if operation != 'tick':
+                    raise AccessError('Pending recovery completed; retry initialization')
+                return {'status': 'rolled_back'}
+            pending = state and state['status'] in ('transition', 'final')
+            if pending and (self.now() >= state['deadline'] or self.host.boot_id() != state['boot_id']):
+                self._rollback(state)
+                if operation != 'tick':
+                    raise AccessError('Uncommitted access transition expired or rebooted; previous access restored')
+                return {'status': 'rolled_back'}
+            if operation == 'tick':
+                return {'status': state['status'] if state else 'absent'}
+            if operation == 'begin':
+                if pending:
+                    raise AccessError('A pending access transition exists; wait for rollback before retrying')
+                if not token or not original_port or not 1 <= original_port <= 65535:
+                    raise AccessError('Access transition requires an original port and token')
+                try:
+                    ipaddress.ip_address(client_address)
+                except ValueError:
+                    raise AccessError('Invalid SSH client address') from None
+                self.host.preflight()
+                path = self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+                return self._begin_manual(token, original_port, client_address, path)
+            if not pending or token != state['token']:
+                raise AccessError('No matching pending access transition')
+            if operation == 'rollback':
+                self._rollback(state)
+                return {'status': 'rolled_back'}
+            if state.get('fingerprint') != self.host.fingerprint():
+                raise AccessError('External SSH configuration changed; finalization stopped, rollback remains armed')
+            if operation == 'finalize':
+                if state['status'] != 'transition':
+                    raise AccessError('Access transition is not ready for final policy')
+                try:
+                    if not state.get('preserve_policy'):
+                        state['ssh_changed'] = True
+                        self.save(state)
+                        self.host.install_policy(self.host.final_ports, True, state['client_address'])
+                    state['authenticated_after_us'] = time.monotonic_ns() // 1000
+                    state['status'] = 'final'
+                    state['fingerprint'] = self.host.fingerprint()
+                    self.save(state)
+                except Exception:
+                    self._rollback(state)
+                    raise
+                return {'status': 'final'}
+            if operation == 'commit' and state['status'] == 'final':
+                self.host.verify_active(self.host.final_ports, True, state['client_address'])
+                state['status'] = 'committed'
+                state['disarm_pending'] = True
+                self.save(state)  # Timer observes this before it can restore anything.
+                self._disarm(state)
+                return {'status': 'committed'}
+            raise AccessError('Invalid access transition operation')
+
+    def _begin_manual(self, token, original_port, address, path):
+        _, pid = self.host.live_arguments()
+        ports = self.host.listener_ports(pid)
+        if original_port not in ports:
+            raise AccessError('Original SSH port is not active on the managed daemon')
+        preserve_policy = False
+        if ports == set(self.host.final_ports):
+            try:
+                self.host.verify_active(self.host.final_ports, True, address)
+                preserve_policy = True
+            except AccessError:
+                pass
+        previous = base64.b64encode(path.read_bytes()).decode() if path.exists() else None
+        state = {'status': 'transition', 'profile': self.profile, 'token': token,
+                 'deadline': self.now() + 300, 'boot_id': self.host.boot_id(),
+                 'bundle': str(Path(__file__).resolve().parents[1]),
+                 'client_address': address, 'original_port': original_port, 'previous': previous,
+                 'preserve_policy': preserve_policy, 'ssh_changed': 2222 not in ports}
+        self.host.arm()
+        self.save(state)
+        try:
+            if 2222 not in ports:
+                self.host.install_policy(sorted(ports | {2222}), False, address)
+            state['fingerprint'] = self.host.fingerprint()
+            state['authenticated_after_us'] = time.monotonic_ns() // 1000
+            self.save(state)
+        except Exception:
+            self._rollback(state)
+            raise
+        return {'status': 'transition'}
+
+    def _rollback(self, state):
+        errors = []
+        if not state.get('ssh_restored') and state.get('ssh_changed', True):
+            try:
+                self.host.restore(state)
+                state['ssh_restored'] = True
+            except Exception:
+                errors.append('ssh')
+        if errors:
+            state['status'] = 'recovery_pending'
+            state['recovery_errors'] = errors
+            self.save(state)
+            raise AccessError('Recovery remains pending for ' + ', '.join(errors) +
+                              '; watchdog remains armed; inspect the root-owned access record')
+        state['status'] = 'rolled_back'
+        state.pop('recovery_errors', None)
+        state['disarm_pending'] = True
+        self.save(state)
+        self._disarm(state)
+
+    def _disarm(self, state):
+        self.host.disarm()
+        state.pop('disarm_pending', None)
+        self.save(state)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('operation', choices=['begin', 'finalize', 'commit', 'rollback', 'tick'])
+    parser.add_argument('--profile', choices=MANUAL_PROFILES)
+    parser.add_argument('--token')
+    parser.add_argument('--original-port', type=int)
+    parser.add_argument('--client-address')
+    args = parser.parse_args(argv)
+    if os.geteuid() != 0:
+        parser.error('Access handover requires root')
+    try:
+        if args.operation != 'tick' and not args.profile:
+            parser.error('A host profile is required')
+        result = Transition(profile=args.profile).execute(args.operation, args.token, args.original_port, args.client_address)
+        print(json.dumps(result))
+        return 0
+    except AccessError as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/provisioning/docker.py

@@ -0,0 +1,301 @@
+"""Verify or install a local Docker Engine and CLI plugins."""
+from __future__ import annotations
+
+from contextlib import contextmanager
+import fcntl
+from functools import wraps
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import tempfile
+import urllib.request
+
+
+class DockerError(RuntimeError):
+    """A required Docker postcondition could not be established."""
+
+
+def expected_errors(operation):
+    @wraps(operation)
+    def guarded(*args, **kwargs):
+        try:
+            return operation(*args, **kwargs)
+        except (OSError, ValueError) as exc:
+            raise DockerError("Docker filesystem or saved state is invalid; inspect the target before retrying.") from exc
+    return guarded
+
+
+DOCKER_PACKAGES = ("docker-ce", "docker-ce-cli", "containerd.io",
+                   "docker-buildx-plugin", "docker-compose-plugin")
+DOCKER_CONFLICTS = ("docker.io", "docker-doc", "docker-compose", "docker-buildx",
+                    "podman-docker", "containerd", "runc")
+DOCKER_CONFIG = {"data-root": "/srv/docker", "log-driver": "json-file",
+                 "log-opts": {"max-size": "20m", "max-file": "3"}}
+POLICY = Path("/usr/sbin/policy-rc.d")
+POLICY_STATE = Path("/usr/sbin/.apex-policy-rc.d")
+APT = ["apt-get", "-o", "DPkg::Lock::Timeout=60", "-o",
+       "Acquire::Retries=2", "-o", "Acquire::http::Timeout=30", "-o",
+       "Acquire::https::Timeout=30"]
+
+
+def run(args, *, check=True, timeout=300):
+    try:
+        result = subprocess.run(args, text=True, capture_output=True, timeout=timeout,
+                                env={**os.environ, "LC_ALL": "C", "DEBIAN_FRONTEND": "noninteractive"})
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        raise DockerError(f"Cannot complete {args[0]}: {exc}") from exc
+    if check and result.returncode:
+        raise DockerError(f"{args[0]} failed with exit status {result.returncode}; inspect target system logs.")
+    return result
+
+
+def package_versions():
+    architecture = run(["dpkg", "--print-architecture"]).stdout.strip()
+    result = run(["dpkg-query", "-W", "-f=${binary:Package}\t${Status}\t${Version}\n"])
+    installed = {}
+    for line in result.stdout.splitlines():
+        fields = line.split("\t")
+        if len(fields) == 3 and fields[1].split()[1:] == ["ok", "installed"]:
+            installed[fields[0].removesuffix(":" + architecture)] = fields[2]
+    return installed
+
+
+def write_managed(path, content, mode=0o644):
+    path = Path(path)
+    for parent in (path, *path.parents):
+        if parent.is_symlink():
+            raise DockerError(f"Refusing managed symlink: {parent}")
+    payload = content.encode() if isinstance(content, str) else content
+    if path.exists() and path.read_bytes() == payload:
+        if path.stat().st_mode & 0o777 != mode:
+            path.chmod(mode)
+        return False
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=path.parent)
+    try:
+        with os.fdopen(fd, "wb") as stream:
+            stream.write(payload)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(temporary, mode)
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+    return True
+
+
+def restore_service_policy():
+    if not POLICY_STATE.exists():
+        return
+    backup = POLICY_STATE / "original"
+    if (POLICY_STATE.is_symlink() or POLICY_STATE.stat().st_uid != os.geteuid()):
+        raise DockerError("Refusing untrusted service policy recovery state.")
+    if (POLICY_STATE / "ready").exists() and POLICY.exists():
+        original_matches = (backup.exists() and not POLICY.is_symlink() and
+                            POLICY.read_bytes() == backup.read_bytes())
+        symlink_matches = (backup.is_symlink() and POLICY.is_symlink() and
+                           os.readlink(backup) == os.readlink(POLICY))
+        guard_matches = not POLICY.is_symlink() and POLICY.read_bytes() == b"#!/bin/sh\nexit 101\n"
+        if not (original_matches or symlink_matches or guard_matches):
+            raise DockerError("Service policy changed outside APEX; recovery requires explicit reconciliation.")
+    if backup.exists() or backup.is_symlink():
+        # Copy first: an interrupted restoration must retain its recovery copy.
+        temporary = POLICY_STATE / "restoring"
+        if temporary.exists() or temporary.is_symlink():
+            temporary.unlink()
+        shutil.copy2(backup, temporary, follow_symlinks=False)
+        os.replace(temporary, POLICY)
+    elif (POLICY_STATE / "absent").exists():
+        POLICY.unlink(missing_ok=True)
+    elif (POLICY_STATE / "ready").exists():
+        raise DockerError("Service policy recovery is incomplete; inspect /usr/sbin/.apex-policy-rc.d.")
+    shutil.rmtree(POLICY_STATE)
+
+
+@contextmanager
+def service_start_guard():
+    lock_path = Path("/run/lock/apex-package-policy.lock")
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a") as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise DockerError("Another APEX package operation is running.") from None
+        with _service_start_guard():
+            yield
+
+
+@contextmanager
+def _service_start_guard():
+    restore_service_policy()
+    POLICY_STATE.mkdir(mode=0o700)
+    try:
+        if POLICY.exists() or POLICY.is_symlink():
+            shutil.copy2(POLICY, POLICY_STATE / "original", follow_symlinks=False)
+        else:
+            (POLICY_STATE / "absent").touch()
+        (POLICY_STATE / "ready").touch()
+        # Replace the path itself, preserving any original symlink in the backup.
+        temporary = POLICY_STATE / "guard"
+        temporary.write_text("#!/bin/sh\nexit 101\n")
+        temporary.chmod(0o755)
+        os.replace(temporary, POLICY)
+        yield
+    finally:
+        restore_service_policy()
+
+
+def install_missing(packages):
+    installed = package_versions()
+    missing = [package for package in packages if package not in installed]
+    if not missing:
+        return []
+    run([*APT, "update", "--error-on=any"])
+    command = [*APT, "install", "-y", "--no-install-recommends", "--no-upgrade", "--no-remove", *missing]
+    plan = run([*command, "--simulate"]).stdout
+    if any(line.startswith("Remv ") or re.match(r"^Inst \S+ \[", line)
+           for line in plan.splitlines()):
+        raise DockerError("Missing packages require an upgrade/removal; reconcile packages explicitly before retrying.")
+    # Let dpkg finish before restoring the service-start policy.
+    run(command, timeout=None)
+    remaining = set(missing) - package_versions().keys()
+    if remaining:
+        raise DockerError("Required packages not installed: " + ", ".join(sorted(remaining)))
+    return missing
+
+
+def docker_preflight(installed):
+    conflicts = sorted(set(DOCKER_CONFLICTS) & installed.keys())
+    if conflicts:
+        raise DockerError("Conflicting Docker packages require explicit reconciliation: " + ", ".join(conflicts))
+    path = Path("/etc/docker/daemon.json")
+    try:
+        config = json.loads(path.read_text()) if path.exists() else {}
+    except (OSError, ValueError) as exc:
+        raise DockerError("Cannot validate existing Docker daemon configuration.") from exc
+    if not isinstance(config, dict):
+        raise DockerError("Docker daemon configuration must be an object.")
+    for name, value in DOCKER_CONFIG.items():
+        if name in config and config[name] != value:
+            raise DockerError(f"Conflicting Docker {name}; reconcile explicitly before initialization.")
+    old_data = Path("/var/lib/docker")
+    if old_data.exists() and any(old_data.iterdir()):
+        raise DockerError("Existing /var/lib/docker data must be reconciled explicitly; no data migration is automatic.")
+    data = Path("/srv/docker")
+    if data.is_symlink() or data.parent.is_symlink():
+        raise DockerError("Refusing a redirected Docker data-root.")
+    if data.exists() and any(data.iterdir()) and config.get("data-root") != "/srv/docker":
+        raise DockerError("Unmanaged /srv/docker data requires explicit reconciliation.")
+    if "docker-ce" in installed:
+        active = run(["systemctl", "is-active", "--quiet", "docker"], check=False).returncode == 0
+        if active:
+            root = run(["docker", "info", "--format", "{{.DockerRootDir}}"] ).stdout.strip()
+            if root != "/srv/docker" or any(config.get(key) != value for key, value in DOCKER_CONFIG.items()):
+                raise DockerError("Running Docker has incompatible data-root/log policy; reconcile it before retrying.")
+    return {**config, **DOCKER_CONFIG}
+
+
+DOCKER_KEY_SHA256 = "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570"
+LOCAL_DOCKER = ["docker", "--host", "unix:///var/run/docker.sock"]
+
+
+def _docker_source(arch):
+    key = Path("/etc/apt/keyrings/apex-docker.asc")
+    source = Path("/etc/apt/sources.list.d/apex-docker.sources")
+    content = ("Types: deb\nURIs: https://download.docker.com/linux/debian\nSuites: trixie\n"
+               f"Components: stable\nArchitectures: {arch}\nSigned-By: {key}\n")
+    for path in (key, source):
+        if any(part.is_symlink() for part in (path, *path.parents)):
+            raise DockerError(f"Refusing redirected APT path: {path}")
+    if source.exists() and source.read_text() != content:
+        raise DockerError(f"Reconcile existing Docker repository configuration: {source}")
+    if key.exists():
+        data = key.read_bytes()
+    else:
+        url = "https://download.docker.com/linux/debian/gpg"
+        with urllib.request.urlopen(url, timeout=30) as response:  # noqa: S310
+            data = response.read(1024 * 1024)
+    if hashlib.sha256(data).hexdigest() != DOCKER_KEY_SHA256:
+        raise DockerError("Docker repository signing key differs from the pinned release key.")
+    write_managed(key, data)
+    write_managed(source, content)
+
+
+def _docker_capabilities(root, *, complete_plugins=False):
+    capabilities = {"docker_data_root": root}
+    plugins = {"docker-compose-plugin": ("compose", ["compose", "version", "--short"]),
+               "docker-buildx-plugin": ("buildx", ["buildx", "version"])}
+    missing = []
+    for package, (name, args) in plugins.items():
+        result = run([*LOCAL_DOCKER, *args], check=False)
+        if result.returncode or not result.stdout.strip():
+            missing.append(package)
+        else:
+            capabilities[name] = result.stdout.strip()
+    if missing:
+        if complete_plugins and "docker-ce" in package_versions():
+            with service_start_guard():
+                install_missing(missing)
+            return _docker_capabilities(root)
+        raise DockerError("Existing Docker needs functional plugins: " + ", ".join(missing))
+    if not root:
+        raise DockerError("Unable to verify the local Docker data-root.")
+    return capabilities
+
+
+@expected_errors
+def apply_docker_policy():
+    if Path("/usr/bin/pveversion").exists():
+        raise DockerError("Docker installation on a Proxmox host requires separate administration.")
+    release = dict(line.split("=", 1) for line in Path("/etc/os-release").read_text().splitlines()
+                   if "=" in line and not line.startswith("#"))
+    arch = run(["dpkg", "--print-architecture"]).stdout.strip()
+    if (release.get("ID", "").strip('"') != "debian" or
+            release.get("VERSION_ID", "").strip('"') != "13" or arch not in ("amd64", "arm64")):
+        raise DockerError("Docker installation supports stock Debian 13 amd64 or arm64.")
+    info = [*LOCAL_DOCKER, "info", "--format", "{{.DockerRootDir}}"]
+    if shutil.which("docker"):
+        current = run(info, check=False)
+        if current.returncode == 0:
+            return _docker_capabilities(current.stdout.strip(), complete_plugins=True)
+    installed = package_versions()
+    source = Path("/etc/apt/sources.list.d/apex-docker.sources")
+    if (not source.exists() and
+            (shutil.which("docker") or set(DOCKER_PACKAGES).intersection(installed))):
+        raise DockerError("Existing Docker installation is inactive; reconcile it before retrying.")
+    config = docker_preflight(installed)
+    with service_start_guard():
+        install_missing(("ca-certificates",))
+        _docker_source(arch)
+        write_managed("/etc/docker/daemon.json", json.dumps(config, indent=2) + "\n")
+        install_missing(DOCKER_PACKAGES)
+    ensure_service("containerd")
+    ensure_service("docker")
+    root = run(info).stdout.strip()
+    if root != DOCKER_CONFIG["data-root"]:
+        raise DockerError("Docker started with an unexpected data-root; inspect its configuration.")
+    return _docker_capabilities(root)
+
+
+def ensure_service(name):
+    if run(["systemctl", "is-enabled", "--quiet", name], check=False).returncode:
+        run(["systemctl", "enable", name])
+    if run(["systemctl", "is-active", "--quiet", name], check=False).returncode:
+        run(["systemctl", "start", name])
+
+
+if __name__ == "__main__":
+    from .state import ProvisionError, exclusive_lock
+
+    try:
+        if os.geteuid() != 0 or not Path("/etc/debian_version").is_file():
+            raise DockerError("Docker setup requires root on Debian.")
+        with exclusive_lock("/var/lib/apex-init/init.lock"):
+            print(json.dumps(apply_docker_policy(), sort_keys=True))
+    except (DockerError, OSError, ProvisionError) as exc:
+        raise SystemExit(f"Docker setup failed: {exc}") from None

scripts/provisioning/manual_bundle.py

@@ -0,0 +1,76 @@
+"""Install an immutable watchdog bundle from the operator's checkout."""
+import hashlib
+import io
+import os
+from pathlib import Path
+import shutil
+import stat
+import tarfile
+import tempfile
+
+
+def _bundle_from_source(root):
+    root = Path(root).resolve()
+    files = [root / 'provisioning' / name for name in ('__init__.py', 'access.py', 'state.py')]
+    output = io.BytesIO()
+    with tarfile.open(fileobj=output, mode='w') as archive:
+        for path in files:
+            if (not path.is_file() or '__pycache__' in path.parts
+                    or any(part.is_symlink() for part in (path, *path.parents))):
+                raise RuntimeError('Provisioning bundle contains an unsafe source path')
+            data = path.read_bytes()
+            info = tarfile.TarInfo(path.relative_to(root).as_posix())
+            info.size, info.mode, info.mtime = len(data), 0o644, 0
+            info.uid = info.gid = 0
+            archive.addfile(info, io.BytesIO(data))
+    data = output.getvalue()
+    return data, hashlib.sha256(data).hexdigest()
+
+
+def _trusted(path):
+    info = path.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o022:
+        raise RuntimeError(f"Untrusted provisioning directory: {path}")
+
+
+def install(root=Path('/')):
+    source = Path(__file__).resolve().parents[1]
+    data, digest = _bundle_from_source(source)
+    # All ancestors from the filesystem root must be protected before publishing code.
+    parent = root
+    _trusted(parent)
+    for component in ('usr', 'local', 'lib', 'apex-provisioner'):
+        parent = parent / component
+        parent.mkdir(mode=0o755, exist_ok=True)
+        _trusted(parent)
+    destination = parent / digest
+    with tempfile.TemporaryDirectory(prefix='.install-', dir=parent) as temporary:
+        stage = Path(temporary) / 'bundle'
+        stage.mkdir(mode=0o755)
+        with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+            for member in archive:
+                if not member.isfile() or '..' in Path(member.name).parts or Path(member.name).is_absolute():
+                    raise RuntimeError('Invalid provisioning bundle member')
+                target = stage / member.name
+                target.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+                with archive.extractfile(member) as source_file, target.open('xb') as output:
+                    shutil.copyfileobj(source_file, output)
+                target.chmod(member.mode)
+        try:
+            os.rename(stage, destination)
+        except OSError:
+            _trusted(destination)
+            installed, existing_digest = _bundle_from_source(destination)
+            if existing_digest != digest or installed != data:
+                raise RuntimeError('Installed provisioning bundle differs from the checkout') from None
+            for path in destination.rglob('*'):
+                info = path.lstat()
+                if info.st_uid != os.geteuid() or info.st_mode & 0o022 or path.is_symlink():
+                    raise RuntimeError(f'Untrusted installed provisioning file: {path}') from None
+    return destination
+
+
+if __name__ == '__main__':
+    if os.geteuid() != 0:
+        raise SystemExit('Run bundle installation as root.')
+    print(install())

scripts/provisioning/state.py

@@ -0,0 +1,37 @@
+"""Serialize host configuration changes across provisioning commands."""
+import contextlib
+import fcntl
+import os
+from pathlib import Path
+import stat
+
+
+class ProvisionError(RuntimeError):
+    pass
+
+
+def safe_path(path):
+    path = Path(path).absolute()
+    for part in (*reversed(path.parents), path):
+        try:
+            info = part.lstat()
+        except FileNotFoundError:
+            continue
+        if stat.S_ISLNK(info.st_mode):
+            raise ProvisionError('managed path contains a symbolic link')
+    return path
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    path = safe_path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+    try:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise ProvisionError('another initialization is running; inspect status before retrying') from None
+        yield
+    finally:
+        os.close(fd)