feat(engine): de-slop commons, harden security, add targeted compose and env bridge

- Purge misplaced RFCs, scratch tokens, and node-proprietary assets
- Decouple remote provisioning to ci/scripts/init-node.py
- Add targeted composition orchestration in actions/compose.py
- Add host-side Xray stale socket cleanup in /dev/shm
- Add actions/env.py for shell exports and .env synchronization
- Harden UFW, backup retention, host tempfiles, and notify dispatcher
- Add pyproject.toml and Makefile with 78 passing unit tests

ci/scripts/init-node/provisioning/__init__.py

@@ -0,0 +1,3 @@
+"""Shared initialization primitives; importing this package performs no host changes."""
+
+ENROLLMENT_INTERFACE = 1

ci/scripts/init-node/provisioning/access.py

@@ -0,0 +1,558 @@
+"""Protected SSH handover with a separately locked, persistent rollback timer."""
+from __future__ import annotations
+
+import argparse
+import base64
+import fcntl
+import fnmatch
+import hashlib
+import ipaddress
+import json
+import os
+from pathlib import Path
+import pwd
+import re
+import shlex
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+
+
+class AccessError(RuntimeError):
+    pass
+
+
+def include_arguments(value):
+    arguments, word, quote, started = [], [], None, False
+    for character in value:
+        if character == '\\':
+            raise AccessError('Escaped SSH Include paths require manual reconciliation')
+        if quote:
+            if character == quote:
+                quote = None
+            else:
+                word.append(character)
+        elif character in ('"', "'"):
+            quote, started = character, True
+        elif character in ' \t\r':
+            if started:
+                arguments.append(''.join(word))
+                word, started = [], False
+        elif character == '#' and not started:
+            break
+        else:
+            word.append(character)
+            started = True
+    if quote:
+        raise AccessError('Unclosed SSH Include quote requires manual reconciliation')
+    if started:
+        arguments.append(''.join(word))
+    if not arguments or len(arguments) > 128 or any(not item or len(item) > 4096 for item in arguments):
+        raise AccessError('Unsupported SSH Include paths require manual reconciliation')
+    return arguments
+
+
+class Host:
+    def __init__(self, root=Path('/'), runner=subprocess.run):
+        self.root = Path(root).resolve()
+        self.runner = runner
+
+    def path(self, path):
+        return self.root / path.lstrip('/')
+
+    def command(self, args, check=True):
+        result = self.runner(args, capture_output=True, text=True, timeout=60)
+        if check and result.returncode:
+            raise AccessError('Access command failed: ' + args[0])
+        return result
+
+    def boot_id(self):
+        return self.path('/proc/sys/kernel/random/boot_id').read_text().strip()
+
+    def fingerprint(self):
+        digest = hashlib.sha256()
+        paths = [self.path('/etc/ssh/sshd_config'), self.path('/etc/default/ssh')]
+        for directory in ('/etc/ssh/sshd_config.d', '/etc/systemd/system/ssh.service.d',
+                          '/run/systemd/system/ssh.service.d', '/usr/lib/systemd/system/ssh.service.d',
+                          '/lib/systemd/system/ssh.service.d'):
+            paths += sorted(self.path(directory).glob('*.conf'))
+        for path in paths:
+            digest.update(str(path.relative_to(self.root)).encode())
+            digest.update(path.read_bytes() if path.exists() else b'<absent>')
+        return digest.hexdigest()
+
+    def unconditional_configuration(self):
+        remaining, directory_entries, visited = 1024 * 1024, 0, set()
+
+        def expand(pattern):
+            nonlocal directory_entries
+            if any(token in pattern for token in ('[:', '[.', '[=', '[^')):
+                raise AccessError('Extended SSH Include glob syntax requires manual reconciliation')
+            absolute = pattern if pattern.startswith('/') else '/etc/ssh/' + pattern
+            mapped = self.path(absolute)
+            if len(mapped.parts) > 64:
+                raise AccessError('SSH Include path exceeds supported limits; manual reconciliation required')
+            candidates = [Path(mapped.anchor)]
+            for component in mapped.parts[1:]:
+                expanded = []
+                for parent in candidates:
+                    if not any(character in component for character in '*?['):
+                        candidate = parent / component
+                        if os.path.lexists(candidate):
+                            expanded.append(candidate)
+                        continue
+                    try:
+                        with os.scandir(parent) as entries:
+                            for entry in entries:
+                                directory_entries += 1
+                                if directory_entries > 4096:
+                                    raise AccessError('SSH Include traversal exceeds supported limits; manual reconciliation required')
+                                if entry.name.startswith('.') and not component.startswith('.'):
+                                    continue
+                                if fnmatch.fnmatchcase(entry.name, component):
+                                    expanded.append(Path(entry.path))
+                    except (FileNotFoundError, NotADirectoryError):
+                        continue
+                if len(expanded) > 128:
+                    raise AccessError('SSH Include expansion exceeds supported limits; manual reconciliation required')
+                candidates = expanded
+            return sorted(candidates)
+
+        def inspect(path, ancestors):
+            nonlocal remaining
+            information = path.stat()
+            identity = (information.st_dev, information.st_ino)
+            if identity in ancestors or len(ancestors) >= 16:
+                raise AccessError('Recursive SSH Include configuration requires manual reconciliation')
+            if identity in visited:
+                return
+            if len(visited) >= 128 or not stat.S_ISREG(information.st_mode) or information.st_size > remaining:
+                raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required')
+            visited.add(identity)
+            with path.open('rb') as stream:
+                content = stream.read(remaining + 1)
+            remaining -= len(content)
+            if remaining < 0:
+                raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required')
+            for line in content.decode().splitlines():
+                line = line.lstrip()
+                if not line or line.startswith('#'):
+                    continue
+                if line.startswith('"'):
+                    keyword, separator, value = line[1:].partition('"')
+                    if not separator:
+                        raise AccessError('Unsupported SSH keyword quoting requires manual reconciliation')
+                else:
+                    parsed = re.match(r'([^\s=]+)(.*)', line)
+                    if not parsed:
+                        raise AccessError('Unsupported SSH configuration syntax requires manual reconciliation')
+                    keyword, value = parsed.groups()
+                    value = value.lstrip()
+                    if value.startswith('='):
+                        value = value[1:]
+                if keyword.lower() == 'match':
+                    raise AccessError('SSH Match directives require manual reconciliation before initialization')
+                if keyword.lower() != 'include':
+                    continue
+                for pattern in include_arguments(value):
+                    for included in expand(pattern):
+                        inspect(included, ancestors | {identity})
+
+        try:
+            inspect(self.path('/etc/ssh/sshd_config'), set())
+        except (OSError, UnicodeError) as exc:
+            raise AccessError('Cannot inspect SSH configuration and includes; manual reconciliation required') from exc
+
+    def bootstrap_preflight(self):
+        for activity in ('is-active', 'is-enabled'):
+            if self.command(['systemctl', activity, '--quiet', 'ssh.socket'], False).returncode == 0:
+                raise AccessError('SSH socket activation requires manual handover; no access changes made')
+        if self.command(['systemctl', 'is-active', '--quiet', 'ssh.service'], False).returncode:
+            raise AccessError('An active ssh.service is required')
+        mode = self.command(['systemctl', 'show', 'ssh.service', '-p', 'KillMode', '--value']).stdout.strip()
+        if mode != 'process':
+            raise AccessError('ssh.service must preserve established sessions with KillMode=process')
+        arguments = self.service_arguments()
+        managed = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        if managed.exists():
+            commands = [line.removeprefix('ExecStart=') for line in managed.read_text().splitlines()
+                        if line.startswith('ExecStart=') and line != 'ExecStart=']
+            if len(commands) != 1 or arguments != shlex.split(commands[0]):
+                raise AccessError('Active SSH service configuration differs from its managed invocation')
+        elif arguments != ['/usr/sbin/sshd', '-D']:
+            raise AccessError('Custom SSH service invocation requires manual access reconciliation')
+        defaults = self.path('/etc/default/ssh')
+        if defaults.exists():
+            for line in defaults.read_text().splitlines():
+                if re.match(r'^\s*SSHD_OPTS\s*=', line):
+                    value = line.split('=', 1)[1].strip().strip('\"\'')
+                    if value:
+                        raise AccessError('Custom SSHD_OPTS require manual access reconciliation')
+        self.unconditional_configuration()
+        self.command(['/usr/sbin/sshd', '-t'])
+        self.live_arguments()
+
+    def preflight(self):
+        self.bootstrap_preflight()
+        self.command(['visudo', '-c'])
+        self.command(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true'])
+
+    def service_arguments(self):
+        output = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout
+        matches = re.findall(r'argv\[\]=([^;]+)', output)
+        if len(matches) != 1:
+            raise AccessError('Unable to verify the SSH service invocation')
+        arguments = shlex.split(matches[0])
+        # Debian's stock unit has this optional variable. Its actual expansion is
+        # independently checked against /proc below, so a nonempty value fails.
+        if arguments == ['/usr/sbin/sshd', '-D', '$SSHD_OPTS']:
+            arguments.pop()
+        if not arguments or arguments[0] != '/usr/sbin/sshd':
+            raise AccessError('SSH service does not invoke the supported daemon')
+        return arguments
+
+    def live_arguments(self):
+        expected = self.service_arguments()
+        pid = self.command(['systemctl', 'show', 'ssh.service', '-p', 'MainPID', '--value']).stdout.strip()
+        if not pid.isdecimal() or int(pid) <= 1:
+            raise AccessError('SSH service has no verifiable active listener process')
+        process = self.path('/proc/' + pid)
+        try:
+            if os.readlink(process / 'exe') != '/usr/sbin/sshd':
+                raise AccessError('Active SSH process does not run the supported daemon')
+            parts = [part for part in (process / 'cmdline').read_bytes().decode().split('\0') if part]
+        except (OSError, UnicodeError):
+            raise AccessError('Unable to inspect the active SSH listener invocation') from None
+        if len(parts) == 1 and parts[0].startswith('sshd: ') and ' [listener]' in parts[0]:
+            actual = shlex.split(parts[0][6:].split(' [listener]', 1)[0])
+        else:
+            actual = parts
+        if actual != expected:
+            raise AccessError('Running SSH listener differs from the active service invocation')
+        return actual, pid
+
+    def verify_active(self, ports, final, address):
+        args, pid = self.live_arguments()
+        self.verify_policy([arg for arg in args if arg != '-D'], ports, final, address)
+        output = self.command(['ss', '-H', '-ltnp']).stdout
+        actual_ports = set()
+        for line in output.splitlines():
+            if '("sshd",' not in line:
+                continue
+            if set(re.findall(r'pid=(\d+)', line)) != {pid}:
+                raise AccessError('An additional SSH listener exists outside the managed service')
+            fields = line.split()
+            if len(fields) < 5 or not fields[3].rsplit(':', 1)[-1].isdecimal():
+                raise AccessError('Unable to verify the active SSH listening sockets')
+            actual_ports.add(int(fields[3].rsplit(':', 1)[-1]))
+        if actual_ports != set(ports):
+            raise AccessError('Active SSH listening ports differ from the declared policy')
+
+    def policy(self, ports, final, address):
+        args = ['/usr/sbin/sshd']
+        for port in sorted(set(ports)):
+            args += ['-p', str(port)]
+        if final:
+            for value in ('PermitRootLogin=no', 'PasswordAuthentication=no',
+                          'KbdInteractiveAuthentication=no', 'PubkeyAuthentication=yes',
+                          'AuthenticationMethods=publickey', 'AllowUsers=adam',
+                          'X11Forwarding=no', 'PermitEmptyPasswords=no'):
+                args += ['-o', value]
+        self.verify_policy(args, ports, final, address)
+        return args
+
+    def verify_policy(self, args, ports, final, address):
+        self.unconditional_configuration()
+        self.command(args + ['-t'])
+        users = {'adam', 'root'}
+        if final:
+            users.update(account.pw_name for account in pwd.getpwall())
+        for user in sorted(users):
+            output = self.command(args + ['-T', '-C', f'user={user},host=localhost,addr={address}']).stdout
+            values = {}
+            for line in output.splitlines():
+                key, _, value = line.partition(' ')
+                values.setdefault(key, []).append(value)
+            if set(values.get('port', [])) != {str(p) for p in ports}:
+                raise AccessError('Effective SSH ports do not match the handover policy')
+            if final and any(values.get(k) != [v] for k, v in {
+                    'permitrootlogin': 'no', 'passwordauthentication': 'no',
+                    'kbdinteractiveauthentication': 'no', 'pubkeyauthentication': 'yes',
+                    'authenticationmethods': 'publickey', 'allowusers': 'adam',
+                    'x11forwarding': 'no', 'permitemptypasswords': 'no'}.items()):
+                raise AccessError('Effective SSH authentication policy is inconsistent')
+
+    def current_final(self, address):
+        try:
+            self.verify_active([2222], True, address)
+            return True
+        except AccessError:
+            return False
+
+    def install_policy(self, ports, final, address):
+        args = self.policy(ports, final, address)
+        path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        atomic(path, ('[Service]\nExecStart=\nExecStart=' + ' '.join(args + ['-D']) + '\n').encode(), 0o644)
+        self.command(['systemctl', 'daemon-reload'])
+        actual = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout
+        match = re.search(r'argv\[\]=([^;]+)', actual)
+        if not match or shlex.split(match[1]) != args + ['-D']:
+            raise AccessError('Another service override prevents the managed SSH policy')
+        self.command(['systemctl', 'restart', 'ssh.service'])
+        self.verify_active(ports, final, address)
+
+    def firewall_needed(self):
+        result = self.command(['ufw', 'status'], False)
+        if result.returncode or 'Status: active' not in result.stdout:
+            return False
+        if re.search(r'^2222/tcp(?:\s|\(v6\)).*ALLOW', result.stdout, re.MULTILINE):
+            return False
+        return True
+
+    def open_firewall(self):
+        # Existing allows remain owned by their administrator. Only our commented rule is removed.
+        self.command(['ufw', 'allow', '2222/tcp', 'comment', 'apex-init-access'])
+
+    def close_firewall(self):
+        self.command(['ufw', '--force', 'delete', 'allow', '2222/tcp', 'comment', 'apex-init-access'])
+
+    def arm(self):
+        bundle = Path(__file__).resolve().parents[2]
+        if not re.fullmatch(r'/usr/local/lib/apex-provisioner/[a-f0-9]{64}', str(bundle)):
+            raise AccessError('Access runner must execute from the installed root-owned bundle')
+        service = ('[Unit]\nDescription=APEX pending access rollback\nAfter=local-fs.target\n'
+                   '[Service]\nType=oneshot\nWorkingDirectory=' + str(bundle) + '\n'
+                   'ExecStart=/usr/bin/python3 -m engine.provisioning.access tick\n')
+        timer = ('[Unit]\nDescription=APEX access rollback deadline and boot recovery\n'
+                 '[Timer]\nOnBootSec=5s\nOnUnitActiveSec=10s\nAccuracySec=1s\n'
+                 '[Install]\nWantedBy=timers.target\n')
+        atomic(self.path('/etc/systemd/system/apex-access-rollback.service'), service.encode(), 0o644)
+        atomic(self.path('/etc/systemd/system/apex-access-rollback.timer'), timer.encode(), 0o644)
+        self.command(['systemctl', 'daemon-reload'])
+        self.command(['systemctl', 'enable', '--now', 'apex-access-rollback.timer'])
+
+    def disarm(self):
+        self.command(['systemctl', 'disable', '--now', 'apex-access-rollback.timer'])
+
+    def restore(self, state):
+        path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        if state['previous'] is None:
+            path.unlink(missing_ok=True)
+        else:
+            atomic(path, base64.b64decode(state['previous']), 0o644)
+        self.command(['/usr/sbin/sshd', '-t'])
+        self.command(['systemctl', 'daemon-reload'])
+        self.command(['systemctl', 'restart', 'ssh.service'])
+        if state.get('firewall_added'):
+            self.close_firewall()
+
+
+def atomic(path, data, mode=0o600):
+    path = Path(path)
+    for parent in [path, *path.parents]:
+        if parent.is_symlink():
+            raise AccessError('Refusing a symlink in a managed access path')
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix='.apex-access-', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'wb') as stream:
+            os.fchmod(stream.fileno(), mode)
+            stream.write(data)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+    directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(directory)
+    finally:
+        os.close(directory)
+
+
+class Transition:
+    def __init__(self, host=None, now=time.monotonic, security=None):
+        from .security import Security
+        self.host = host or Host()
+        self.security = security if security is not None else Security(self.host)
+        self.now = now
+        self.directory = self.host.path('/var/lib/apex-init/access')
+
+    def load(self):
+        path = self.directory / 'state.json'
+        return json.loads(path.read_text()) if path.exists() else None
+
+    def save(self, state):
+        atomic(self.directory / 'state.json', json.dumps(state, sort_keys=True).encode())
+
+    def execute(self, operation, token=None, original_port=None, client_address=None):
+        # Rollback must never wait for a stalled enrollment stage holding init.lock.
+        if operation in ('tick', 'rollback'):
+            return self._execute(operation, token, original_port, client_address)
+        from .state import ProvisionError, exclusive_lock
+        try:
+            with exclusive_lock(self.host.path('/var/lib/apex-init/init.lock')):
+                return self._execute(operation, token, original_port, client_address)
+        except ProvisionError as error:
+            raise AccessError(str(error)) from None
+
+    def _execute(self, operation, token=None, original_port=None, client_address=None):
+        if token is not None and not re.fullmatch(r'[a-f0-9]{32}', token):
+            raise AccessError('Invalid access transition token')
+        for path in [self.directory, *self.directory.parents]:
+            if path.is_symlink():
+                raise AccessError('Invalid access state directory')
+        self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+        if self.directory.stat().st_uid != os.geteuid():
+            raise AccessError('Access state directory is not owned by the provisioner user')
+        os.chmod(self.directory, 0o700)
+        descriptor = os.open(self.directory / 'lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+        with os.fdopen(descriptor, 'w') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            state = self.load()
+            if state and state['status'] == 'recovery_pending':
+                self._rollback(state)
+                if operation != 'tick':
+                    raise AccessError('Pending recovery completed; retry initialization')
+                return {'status': 'rolled_back'}
+            pending = state and state['status'] in ('transition', 'final')
+            if pending and (self.now() >= state['deadline'] or self.host.boot_id() != state['boot_id']):
+                self._rollback(state)
+                if operation != 'tick':
+                    raise AccessError('Uncommitted access transition expired or rebooted; previous access restored')
+                return {'status': 'rolled_back'}
+            if operation == 'tick':
+                return {'status': state['status'] if state else 'absent'}
+            if operation == 'begin':
+                if pending:
+                    raise AccessError('A pending access transition exists; wait for rollback before retrying')
+                if not token or not original_port or not 1 <= original_port <= 65535:
+                    raise AccessError('Access transition requires an original port and token')
+                try:
+                    ipaddress.ip_address(client_address)
+                except ValueError:
+                    raise AccessError('Invalid SSH client address') from None
+                self.host.preflight()
+                path = self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+                preserve_policy = False
+                if state and state['status'] == 'committed':
+                    if state['fingerprint'] != self.host.fingerprint():
+                        raise AccessError('Committed access configuration changed; reconcile before retrying')
+                    self.host.verify_active([2222], True, client_address)
+                    if state.get('security'):
+                        self.security.verify(state['security'])
+                        return {'status': 'committed'}
+                    preserve_policy = True
+                if original_port == 2222 and not path.exists() and self.host.current_final(client_address):
+                    preserve_policy = True
+                security_state = self.security.snapshot(self.security.preflight(), original_port, client_address)
+                self.host.arm()
+                state = {'status': 'transition', 'token': token, 'deadline': self.now() + 300,
+                         'boot_id': self.host.boot_id(), 'client_address': client_address,
+                         'original_port': original_port, 'firewall_added': False,
+                         'preserve_policy': preserve_policy, 'security': security_state,
+                         'previous': base64.b64encode(path.read_bytes()).decode() if path.exists() else None}
+                self.save(state)
+                try:
+                    self.security.activate(state['security'], lambda value: self._save_security(state, value))
+                    if not preserve_policy:
+                        self.host.install_policy([original_port, 2222], False, client_address)
+                    state['fingerprint'] = self.host.fingerprint()
+                    self.save(state)
+                except Exception:
+                    self._rollback(state)
+                    raise
+                return {'status': 'transition'}
+            if not state or (not pending and state['status'] != 'existing') or token != state['token']:
+                raise AccessError('No matching pending access transition')
+            if state['status'] == 'existing':
+                if operation == 'rollback':
+                    state['status'] = 'rolled_back'
+                    self.save(state)
+                    return {'status': 'rolled_back'}
+                raise AccessError('Legacy access verification lacks security activation; retry initialization')
+            if operation == 'rollback':
+                self._rollback(state)
+                return {'status': 'rolled_back'}
+            if state.get('fingerprint') != self.host.fingerprint():
+                raise AccessError('External SSH configuration changed; finalization stopped, rollback remains armed')
+            if operation == 'finalize':
+                if state['status'] != 'transition':
+                    raise AccessError('Access transition is not ready for final policy')
+                try:
+                    if not state.get('preserve_policy'):
+                        self.host.install_policy([2222], True, state['client_address'])
+                    self.security.finalize(state['security'], lambda value: self._save_security(state, value))
+                    state['status'] = 'final'
+                    state['fingerprint'] = self.host.fingerprint()
+                    self.save(state)
+                except Exception:
+                    self._rollback(state)
+                    raise
+                return {'status': 'final'}
+            if operation == 'commit' and state['status'] == 'final':
+                self.host.verify_active([2222], True, state['client_address'])
+                self.security.verify(state['security'])
+                state['status'] = 'committed'
+                self.save(state)  # Timer observes this before it can restore anything.
+                self.host.disarm()
+                return {'status': 'committed'}
+            raise AccessError('Invalid access transition operation')
+
+    def _save_security(self, state, value):
+        state['security'] = value
+        self.save(state)
+
+    def _rollback(self, state):
+        errors = []
+        if state.get('security'):
+            try:
+                self.security.restore(state['security'], lambda value: self._save_security(state, value))
+            except Exception:
+                errors.append('security')
+        if not state.get('ssh_restored'):
+            try:
+                self.host.restore(state)
+                state['ssh_restored'] = True
+            except Exception:
+                errors.append('ssh')
+        if errors:
+            state['status'] = 'recovery_pending'
+            state['recovery_errors'] = errors
+            self.save(state)
+            raise AccessError('Recovery remains pending for ' + ', '.join(errors) +
+                              '; watchdog remains armed; inspect the root-owned access record')
+        state['status'] = 'rolled_back'
+        state.pop('recovery_errors', None)
+        self.save(state)
+        self.host.disarm()
+
+
+def main(argv=None):
+    from .security import SecurityError
+    parser = argparse.ArgumentParser()
+    parser.add_argument('operation', choices=['preflight', 'begin', 'finalize', 'commit', 'rollback', 'tick'])
+    parser.add_argument('--token')
+    parser.add_argument('--original-port', type=int)
+    parser.add_argument('--client-address')
+    args = parser.parse_args(argv)
+    if os.geteuid() != 0:
+        parser.error('Access handover requires root')
+    try:
+        if args.operation == 'preflight':
+            Host().bootstrap_preflight()
+            result = {'status': 'prerequisites-verified'}
+        else:
+            result = Transition().execute(args.operation, args.token, args.original_port, args.client_address)
+        print(json.dumps(result))
+        return 0
+    except (AccessError, SecurityError) as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

ci/scripts/init-node/provisioning/baseline.py

@@ -0,0 +1,487 @@
+"""Shared Debian baseline. Callers own the exclusive initialization lock."""
+from __future__ import annotations
+
+from contextlib import contextmanager
+import fcntl
+from functools import wraps
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import stat
+import subprocess
+import tempfile
+import urllib.request
+import uuid
+
+
+class BaselineError(RuntimeError):
+    """A required baseline postcondition could not be established."""
+
+
+def expected_errors(operation):
+    @wraps(operation)
+    def guarded(*args, **kwargs):
+        try:
+            return operation(*args, **kwargs)
+        except (OSError, ValueError) as exc:
+            raise BaselineError("Baseline filesystem or saved state is invalid; inspect the target before retrying.") from exc
+    return guarded
+
+
+BASE_PACKAGES = ("curl", "wget", "git", "rsyslog", "ufw")
+ADMIN_PACKAGES = (
+    "tcpdump", "socat", "rsync", "bind9-dnsutils", "iproute2", "net-tools",
+    "traceroute", "ca-certificates", "jq", "sed", "gawk", "tar", "gzip",
+    "bzip2", "xz-utils", "unzip", "htop", "iotop", "strace", "lsof",
+    "sysstat", "parted", "fdisk", "pciutils", "usbutils", "smartmontools",
+    "tmux", "vim", "less", "tree", "bash-completion", "python3",
+    "python3-minimal", "python3-yaml", "sudo", "cron", "iptables", "ipset",
+    "debian-archive-keyring", "kmod",
+)
+DOCKER_PACKAGES = ("docker-ce", "docker-ce-cli", "containerd.io",
+                   "docker-buildx-plugin", "docker-compose-plugin")
+SECURITY_PACKAGES = ("crowdsec", "crowdsec-firewall-bouncer-iptables")
+DOCKER_CONFLICTS = ("docker.io", "docker-doc", "docker-compose", "docker-buildx",
+                    "podman-docker", "containerd", "runc")
+DOCKER_CONFIG = {"data-root": "/srv/docker", "log-driver": "json-file",
+                 "log-opts": {"max-size": "20m", "max-file": "3"}}
+HELPER_REVISION = "020a8699f95592561f254d8d4ad1bb40d401dfc7"
+HELPER_URL = f"https://raw.githubusercontent.com/chaifeng/ufw-docker/{HELPER_REVISION}/ufw-docker"
+HELPER_SHA256 = "643e56b080567c567b4aa28650196849b2a2da5dd0473fd3e5216b0886035ab0"
+POLICY = Path("/usr/sbin/policy-rc.d")
+POLICY_STATE = Path("/usr/sbin/.apex-policy-rc.d")
+SECURITY_STATE = Path("/var/lib/apex/baseline-security.json")
+SECURITY_SERVICES = {"ufw": "ufw", "crowdsec": "crowdsec",
+                     "crowdsec-firewall-bouncer-iptables": "crowdsec-firewall-bouncer"}
+SECURITY_CONDITION = "[Unit]\nConditionPathExists=/dev/null/apex-security-start-allowed\n"
+APT = ["apt-get", "-o", "DPkg::Lock::Timeout=60", "-o",
+       "Acquire::Retries=2", "-o", "Acquire::http::Timeout=30", "-o",
+       "Acquire::https::Timeout=30"]
+
+
+def run(args, *, check=True, timeout=300):
+    try:
+        result = subprocess.run(args, text=True, capture_output=True, timeout=timeout,
+                                env={**os.environ, "LC_ALL": "C", "DEBIAN_FRONTEND": "noninteractive"})
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        raise BaselineError(f"Cannot complete {args[0]}: {exc}") from exc
+    if check and result.returncode:
+        raise BaselineError(f"{args[0]} failed with exit status {result.returncode}; inspect target system logs.")
+    return result
+
+
+def preflight(*, strict=True):
+    if os.geteuid() != 0:
+        raise BaselineError("Baseline requires root.")
+    if not strict:
+        if not Path("/etc/debian_version").is_file():
+            raise BaselineError("Base policy requires a Debian-based system.")
+        return
+    if not Path("/etc/os-release").is_file():
+        raise BaselineError("Supported target is Debian 13 amd64 only.")
+    release = dict(line.split("=", 1) for line in Path("/etc/os-release").read_text().splitlines()
+                   if "=" in line and not line.startswith("#"))
+    if (release.get("ID", "").strip('"') != "debian" or
+            release.get("VERSION_ID", "").strip('"') != "13" or
+            run(["dpkg", "--print-architecture"]).stdout.strip() != "amd64"):
+        raise BaselineError("Supported target is Debian 13 amd64 only.")
+
+
+def package_versions():
+    result = run(["dpkg-query", "-W", "-f=${binary:Package}\t${Status}\t${Version}\n"])
+    installed = {}
+    for line in result.stdout.splitlines():
+        fields = line.split("\t")
+        if len(fields) == 3 and fields[1].split()[1:] == ["ok", "installed"]:
+            installed[fields[0].removesuffix(":amd64")] = fields[2]
+    return installed
+
+
+def write_managed(path, content, mode=0o644):
+    path = Path(path)
+    for parent in (path, *path.parents):
+        if parent.is_symlink():
+            raise BaselineError(f"Refusing managed symlink: {parent}")
+    payload = content.encode() if isinstance(content, str) else content
+    if path.exists() and path.read_bytes() == payload:
+        if path.stat().st_mode & 0o777 != mode:
+            path.chmod(mode)
+        return False
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=path.parent)
+    try:
+        with os.fdopen(fd, "wb") as stream:
+            stream.write(payload)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(temporary, mode)
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+    return True
+
+
+def restore_service_policy():
+    if not POLICY_STATE.exists():
+        return
+    backup = POLICY_STATE / "original"
+    if (POLICY_STATE.is_symlink() or POLICY_STATE.stat().st_uid != os.geteuid()):
+        raise BaselineError("Refusing untrusted service policy recovery state.")
+    if (POLICY_STATE / "ready").exists() and POLICY.exists():
+        original_matches = (backup.exists() and not POLICY.is_symlink() and
+                            POLICY.read_bytes() == backup.read_bytes())
+        symlink_matches = (backup.is_symlink() and POLICY.is_symlink() and
+                           os.readlink(backup) == os.readlink(POLICY))
+        guard_matches = not POLICY.is_symlink() and POLICY.read_bytes() == b"#!/bin/sh\nexit 101\n"
+        if not (original_matches or symlink_matches or guard_matches):
+            raise BaselineError("Service policy changed outside APEX; recovery requires explicit reconciliation.")
+    if backup.exists() or backup.is_symlink():
+        # Copy first: an interrupted restoration must retain its recovery copy.
+        temporary = POLICY_STATE / "restoring"
+        if temporary.exists() or temporary.is_symlink():
+            temporary.unlink()
+        shutil.copy2(backup, temporary, follow_symlinks=False)
+        os.replace(temporary, POLICY)
+    elif (POLICY_STATE / "absent").exists():
+        POLICY.unlink(missing_ok=True)
+    elif (POLICY_STATE / "ready").exists():
+        raise BaselineError("Service policy recovery is incomplete; inspect /usr/sbin/.apex-policy-rc.d.")
+    shutil.rmtree(POLICY_STATE)
+
+
+@contextmanager
+def service_start_guard():
+    lock_path = Path("/run/lock/apex-package-policy.lock")
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a") as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise BaselineError("Another APEX package operation is running.") from None
+        with _service_start_guard():
+            yield
+
+
+@contextmanager
+def _service_start_guard():
+    restore_service_policy()
+    POLICY_STATE.mkdir(mode=0o700)
+    try:
+        if POLICY.exists() or POLICY.is_symlink():
+            shutil.copy2(POLICY, POLICY_STATE / "original", follow_symlinks=False)
+        else:
+            (POLICY_STATE / "absent").touch()
+        (POLICY_STATE / "ready").touch()
+        # Replace the path itself, preserving any original symlink in the backup.
+        temporary = POLICY_STATE / "guard"
+        temporary.write_text("#!/bin/sh\nexit 101\n")
+        temporary.chmod(0o755)
+        os.replace(temporary, POLICY)
+        yield
+    finally:
+        restore_service_policy()
+
+
+@contextmanager
+def security_service_guard(packages):
+    """Guard new security units without occupying package-owned unit paths."""
+    record = json.loads(SECURITY_STATE.read_text()) if SECURITY_STATE.exists() else {"version": 2, "services": {}}
+    if not isinstance(record, dict) or record.get("version") != 2:
+        raise BaselineError("Legacy security service guard state requires explicit reconciliation.")
+    managed = record.get("services")
+    if not isinstance(managed, dict) or any(SECURITY_SERVICES.get(key) != value for key, value in managed.items()):
+        raise BaselineError("Invalid baseline security service recovery state.")
+    installed = package_versions()
+    for package, service in SECURITY_SERVICES.items():
+        if package in packages and package not in installed and package not in managed:
+            if run(["systemctl", "is-enabled", service], check=False).stdout.strip() == "masked":
+                continue
+            managed[package] = service
+            write_managed(SECURITY_STATE, json.dumps({"version": 2, "services": managed}), 0o600)
+    for service in managed.values():
+        dropin = Path(f"/etc/systemd/system/{service}.service.d/99-apex-bootstrap-guard.conf")
+        if dropin.exists() and dropin.read_text() != SECURITY_CONDITION:
+            raise BaselineError("Security service guard changed outside APEX; reconcile it explicitly.")
+        # /dev/null cannot have children: this condition stays false after reboot.
+        write_managed(dropin, SECURITY_CONDITION)
+    if managed:
+        run(["systemctl", "daemon-reload"])
+    try:
+        yield
+    finally:
+        installed = package_versions()
+        for package, service in managed.items():
+            unit_exists = any(Path(f"{directory}/{service}.service").exists() for directory in
+                              ("/etc/systemd/system", "/usr/lib/systemd/system", "/lib/systemd/system"))
+            run(["systemctl", "disable", service], check=package in installed or unit_exists)
+            dropin = Path(f"/etc/systemd/system/{service}.service.d/99-apex-bootstrap-guard.conf")
+            if dropin.exists():
+                if dropin.is_symlink() or dropin.read_text() != SECURITY_CONDITION:
+                    raise BaselineError("Security service guard changed outside APEX; reconcile it explicitly.")
+                dropin.unlink()
+        if managed:
+            run(["systemctl", "daemon-reload"])
+        SECURITY_STATE.unlink(missing_ok=True)
+
+
+def install_missing(packages):
+    installed = package_versions()
+    missing = [package for package in packages if package not in installed]
+    if not missing:
+        return []
+    run([*APT, "update", "--error-on=any"])
+    command = [*APT, "install", "-y", "--no-install-recommends", "--no-upgrade", "--no-remove", *missing]
+    plan = run([*command, "--simulate"]).stdout
+    if any(line.startswith("Remv ") or re.match(r"^Inst \S+ \[", line)
+           for line in plan.splitlines()):
+        raise BaselineError("Missing packages require an upgrade/removal; reconcile packages explicitly before retrying.")
+    # Package writes must settle even after coordinator loss; stages gate later work.
+    run(command, timeout=None)
+    remaining = set(missing) - package_versions().keys()
+    if remaining:
+        raise BaselineError("Required packages not installed: " + ", ".join(sorted(remaining)))
+    return missing
+
+
+def base_policy():
+    return {"packages": list(BASE_PACKAGES), "files": {
+        "/etc/apt/apt.conf.d/99-apex": 'APT::Keep-Downloaded-Packages "0";\n',
+        "/etc/systemd/journald.conf.d/99-apex.conf": "[Journal]\nSystemMaxUse=200M\n",
+    }, "helper": {"url": HELPER_URL, "sha256": HELPER_SHA256}}
+
+
+def configure_common():
+    policy = base_policy()
+    for legacy in ("/etc/apt/apt.conf.d/01clean", "/etc/systemd/journald.conf.d/maxuse.conf"):
+        path = Path(legacy)
+        if path.exists() or path.is_symlink():
+            path.unlink()
+    for path, content in policy["files"].items():
+        changed = write_managed(path, content)
+        if changed and path.endswith("99-apex.conf"):
+            run(["systemctl", "restart", "systemd-journald"])
+    target = Path("/usr/local/bin/ufw-docker")
+    if (not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest() != HELPER_SHA256):
+        try:
+            with urllib.request.urlopen(HELPER_URL, timeout=30) as response:
+                data = response.read()
+        except OSError as exc:
+            raise BaselineError("Unable to download pinned ufw-docker helper.") from exc
+        if hashlib.sha256(data).hexdigest() != HELPER_SHA256:
+            raise BaselineError("Pinned ufw-docker checksum mismatch.")
+        write_managed(target, data, 0o755)
+    elif target.is_symlink():
+        raise BaselineError("Refusing a symlinked ufw-docker helper.")
+    else:
+        target.chmod(0o755)
+
+
+def docker_preflight(installed):
+    conflicts = sorted(set(DOCKER_CONFLICTS) & installed.keys())
+    if conflicts:
+        raise BaselineError("Conflicting Docker packages require explicit reconciliation: " + ", ".join(conflicts))
+    path = Path("/etc/docker/daemon.json")
+    try:
+        config = json.loads(path.read_text()) if path.exists() else {}
+    except (OSError, ValueError) as exc:
+        raise BaselineError("Cannot validate existing Docker daemon configuration.") from exc
+    if not isinstance(config, dict):
+        raise BaselineError("Docker daemon configuration must be an object.")
+    for name, value in DOCKER_CONFIG.items():
+        if name in config and config[name] != value:
+            raise BaselineError(f"Conflicting Docker {name}; reconcile explicitly before initialization.")
+    old_data = Path("/var/lib/docker")
+    if old_data.exists() and any(old_data.iterdir()):
+        raise BaselineError("Existing /var/lib/docker data must be reconciled explicitly; no data migration is automatic.")
+    data = Path("/srv/docker")
+    if data.is_symlink() or data.parent.is_symlink():
+        raise BaselineError("Refusing a redirected Docker data-root.")
+    if data.exists() and any(data.iterdir()) and config.get("data-root") != "/srv/docker":
+        raise BaselineError("Unmanaged /srv/docker data requires explicit reconciliation.")
+    if "docker-ce" in installed:
+        active = run(["systemctl", "is-active", "--quiet", "docker"], check=False).returncode == 0
+        if active:
+            root = run(["docker", "info", "--format", "{{.DockerRootDir}}"] ).stdout.strip()
+            if root != "/srv/docker" or any(config.get(key) != value for key, value in DOCKER_CONFIG.items()):
+                raise BaselineError("Running Docker has incompatible data-root/log policy; reconcile it before retrying.")
+    return {**config, **DOCKER_CONFIG}
+
+
+def configure_sources():
+    write_managed("/etc/apt/sources.list.d/apex-zfs.sources",
+                  "Types: deb\nURIs: https://deb.debian.org/debian\nSuites: trixie\n"
+                  "Components: contrib\nArchitectures: amd64\n"
+                  "Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg\n")
+    for name, key_url, uri, suite, component in (
+        ("docker", "https://download.docker.com/linux/debian/gpg",
+         "https://download.docker.com/linux/debian", "trixie", "stable"),
+        ("crowdsec", "https://packagecloud.io/crowdsec/crowdsec/gpgkey",
+         "https://packagecloud.io/crowdsec/crowdsec/any", "any", "main"),
+    ):
+        key = Path(f"/etc/apt/keyrings/apex-{name}.asc")
+        if not key.exists():
+            try:
+                with urllib.request.urlopen(key_url, timeout=30) as response:
+                    data = response.read()
+            except OSError as exc:
+                raise BaselineError(f"Cannot download {name} repository signing key.") from exc
+            if not data.startswith(b"-----BEGIN PGP PUBLIC KEY BLOCK-----"):
+                raise BaselineError(f"Invalid {name} repository signing key.")
+            write_managed(key, data)
+        write_managed(f"/etc/apt/sources.list.d/apex-{name}.sources",
+                      f"Types: deb\nURIs: {uri}\nSuites: {suite}\nComponents: {component}\n"
+                      f"Architectures: amd64\nSigned-By: {key}\n")
+
+
+def ensure_service(name):
+    if run(["systemctl", "is-enabled", "--quiet", name], check=False).returncode:
+        run(["systemctl", "enable", name])
+    if run(["systemctl", "is-active", "--quiet", name], check=False).returncode:
+        run(["systemctl", "start", name])
+
+
+def repair_bouncer_credentials():
+    """Repair package placeholders without rewriting administrator YAML settings."""
+    config = Path("/etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml")
+    identity = config.with_suffix(".yaml.id")
+    for path in (config, identity):
+        if any(parent.is_symlink() for parent in (path, *path.parents)):
+            raise BaselineError("Refusing redirected CrowdSec bouncer credential paths.")
+        if path.exists():
+            metadata = path.stat()
+            if (not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1 or
+                    metadata.st_uid != os.geteuid()):
+                raise BaselineError("Refusing untrusted CrowdSec bouncer credential files.")
+    if not config.exists():
+        raise BaselineError("CrowdSec bouncer configuration is missing; reconcile the package installation.")
+    content = config.read_text()
+    entries = list(re.finditer(r"^(?:api_key|'api_key'|\"api_key\"):[^\r\n]*", content, re.MULTILINE))
+    if len(entries) != 1:
+        raise BaselineError("CrowdSec bouncer requires one explicit api_key; reconcile its configuration.")
+    entry = entries[0]
+    for following in content[entry.end():].splitlines():
+        if not following.strip() or following.lstrip().startswith("#"):
+            continue
+        if following[:1].isspace():
+            raise BaselineError("Multiline CrowdSec bouncer api_key requires explicit reconciliation.")
+        break
+    scalar = re.fullmatch(r"(?:api_key|'api_key'|\"api_key\"):[ \t]*(?P<value>\"[^\"]*\"|'[^']*'|[^ \t#'\"]*)(?:[ \t]+(?:#.*)?)?", entry.group())
+    if scalar is None:
+        raise BaselineError("Unsupported CrowdSec bouncer api_key; reconcile its configuration.")
+    value = scalar.group("value")
+    uninitialized = value in ("null", "Null", "NULL", "~")
+    if value[:1] in ("'", '"'):
+        value = value[1:-1]
+    if not uninitialized and value not in ("", "<API_KEY>", "$API_KEY", "${API_KEY}"):
+        return
+    metadata = config.stat()
+    registration = "cs-firewall-bouncer-apex-" + uuid.uuid4().hex
+    key = run(["cscli", "-oraw", "bouncers", "add", registration]).stdout.strip()
+    if not re.fullmatch(r"[A-Za-z0-9_+/=-]{24,256}", key):
+        raise BaselineError("CrowdSec bouncer registration did not return a usable API key.")
+    start, end = scalar.span("value")
+    separator = " " if entry.group()[start - 1] == ":" else ""
+    updated = content[:entry.start() + start] + separator + key + content[entry.start() + end:]
+
+    def publish(path, payload, mode, owner):
+        fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=path.parent)
+        try:
+            with os.fdopen(fd, "w") as stream:
+                os.fchown(stream.fileno(), *owner)
+                os.fchmod(stream.fileno(), mode)
+                stream.write(payload)
+                stream.flush()
+                os.fsync(stream.fileno())
+            os.replace(temporary, path)
+            directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
+            try:
+                os.fsync(directory)
+            finally:
+                os.close(directory)
+        finally:
+            if os.path.exists(temporary):
+                os.unlink(temporary)
+
+    # Publish the key last: interruption can orphan a registration, but retries
+    # replace the placeholder and its package bookkeeping together in this order.
+    owner = (metadata.st_uid, metadata.st_gid)
+    publish(identity, registration + "\n", 0o600, owner)
+    publish(config, updated, (stat.S_IMODE(metadata.st_mode) & 0o640) | 0o600, owner)
+    if config.read_text() != updated or identity.read_text().strip() != registration:
+        raise BaselineError("CrowdSec bouncer credential publication did not converge.")
+
+
+@expected_errors
+def apply_base_policy():
+    """The bounded configure/base policy: no Docker, storage or enrollment."""
+    preflight(strict=False)
+    with service_start_guard(), security_service_guard(BASE_PACKAGES):
+        install_missing(BASE_PACKAGES)
+    configure_common()
+    ensure_service("rsyslog")
+    run(["/usr/local/bin/ufw-docker", "install"])
+    return {"packages": {key: value for key, value in package_versions().items() if key in BASE_PACKAGES}}
+
+
+@expected_errors
+def apply_baseline():
+    """Converge baseline and return verified package versions and capabilities."""
+    preflight()
+    installed = package_versions()
+    docker_config = docker_preflight(installed)
+    kernel = run(["uname", "-r"]).stdout.strip()
+    if not re.fullmatch(r"[A-Za-z0-9.+_-]+", kernel):
+        raise BaselineError("Unsupported running kernel release.")
+    packages = (*BASE_PACKAGES, *ADMIN_PACKAGES, *DOCKER_PACKAGES,
+                *SECURITY_PACKAGES, "linux-headers-amd64", f"linux-headers-{kernel}",
+                "zfsutils-linux", "zfs-dkms")
+    with service_start_guard(), security_service_guard(packages):
+        install_missing((*BASE_PACKAGES, *ADMIN_PACKAGES))
+        configure_sources()
+        install_missing(("crowdsec",))
+        install_missing(packages)
+        repair_bouncer_credentials()
+    configure_common()
+    write_managed("/etc/docker/daemon.json", json.dumps(docker_config, indent=2) + "\n")
+    Path("/srv/docker").mkdir(parents=True, exist_ok=True)
+    ensure_service("containerd")
+    ensure_service("docker")
+    ensure_service("rsyslog")
+    ensure_service("cron")
+    root = run(["docker", "info", "--format", "{{.DockerRootDir}}"] ).stdout.strip()
+    if root != "/srv/docker":
+        raise BaselineError("Docker did not activate the required /srv/docker data-root.")
+    compose = run(["docker", "compose", "version", "--short"]).stdout.strip()
+    if not compose:
+        raise BaselineError("Docker Compose capability is unavailable.")
+    module = run(["modprobe", "zfs"], check=False)
+    if module.returncode:
+        raise BaselineError(f"ZFS unavailable for running kernel {kernel}; install matching headers/module or reboot into the supported kernel, then retry. No automatic reboot was performed.")
+    zfs = run(["zfs", "version"]).stdout.strip()
+    versions = package_versions()
+    if set(packages) - versions.keys():
+        raise BaselineError("Baseline package verification failed.")
+    return {"packages": {name: versions[name] for name in packages},
+            "capabilities": {"docker_data_root": root, "compose": compose, "zfs": zfs,
+                             "kernel": kernel, "ufw_docker_sha256": HELPER_SHA256},
+            "sources": {"docker": "docker.com/debian trixie stable",
+                        "crowdsec": "packagecloud.io/crowdsec/crowdsec/any any main",
+                        "zfs": "Debian trixie contrib"}}
+
+
+if __name__ == "__main__":
+    import argparse
+    from .state import ProvisionError, exclusive_lock
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-policy", action="store_true")
+    args = parser.parse_args()
+    try:
+        preflight(strict=not args.base_policy)
+        with exclusive_lock("/var/lib/apex-init/init.lock"):
+            print(json.dumps(apply_base_policy() if args.base_policy else apply_baseline(), sort_keys=True))
+    except (BaselineError, OSError, ProvisionError) as exc:
+        parser.exit(1, f"Baseline failed: {exc}\n")

ci/scripts/init-node/provisioning/contract.py

@@ -0,0 +1,236 @@
+"""Pure validation for the supported APEX enrollment contract."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import PurePosixPath
+from typing import Any, Dict, Mapping, Optional, Tuple
+from urllib.parse import urlsplit
+
+
+class ContractError(ValueError):
+    """A safe diagnostic that never includes a supplied value."""
+
+
+@dataclass(frozen=True)
+class Repository:
+    name: str
+    url: str
+    ref: Optional[str]
+
+
+@dataclass(frozen=True)
+class Tier:
+    number: int
+    mode: str
+    device: Optional[str]
+    pool_guid: Optional[str]
+
+
+@dataclass(frozen=True)
+class ManagedFile:
+    path: str
+    owner: str
+    mode: int
+    content: str = field(repr=False)
+
+
+@dataclass(frozen=True)
+class Enrollment:
+    hostname: str
+    repository: Repository
+    tiers: Tuple[Tier, ...]
+    authorized_keys: Tuple[str, ...] = field(repr=False)
+    files: Tuple[ManagedFile, ...] = field(repr=False)
+
+    def identity(self) -> Dict[str, Any]:
+        return {
+            'hostname': self.hostname,
+            'repository': {
+                'name': self.repository.name,
+                'url': self.repository.url,
+                'ref': self.repository.ref,
+            },
+            'tiers': [
+                {'number': t.number, 'mode': t.mode,
+                 'device': t.device, 'pool_guid': t.pool_guid}
+                for t in self.tiers
+            ],
+        }
+
+
+_ENV_KEYS = {'APEX_REPO_NAME', 'APEX_REPO_URL', 'APEX_REPO_REF'} | {
+    f'APEX_TIER{number}_{suffix}'
+    for number in (1, 2, 3)
+    for suffix in ('DEVICE', 'MODE', 'POOL_GUID')
+}
+_ASSIGNMENT = re.compile(
+    r'''\s*([A-Z][A-Z0-9_]*)\s*=\s*(?:"([^"\\]*)"|'([^'\\]*)'|([^\s'"\\()#]*))\s*'''
+)
+_FILE_POLICY = {
+    '/etc/apex/bootstrap.env': ('root:root', 0o600, False),
+    '/home/adam/.ssh/id_ed25519': ('adam:adam', 0o600, True),
+    '/home/adam/.ssh/known_hosts': ('adam:adam', 0o644, True),
+}
+
+
+def _require(condition: bool, diagnostic: str) -> None:
+    if not condition:
+        raise ContractError(diagnostic)
+
+
+def _mapping(value: Any, required: set, optional: set, diagnostic: str) -> Mapping:
+    _require(isinstance(value, Mapping), diagnostic)
+    _require(set(value) >= required and set(value) <= required | optional, diagnostic)
+    return value
+
+
+def _plain(value: Any, diagnostic: str, allow_empty: bool = False) -> str:
+    _require(type(value) is str, diagnostic)
+    _require(allow_empty or bool(value), diagnostic)
+    _require(not any(ord(c) < 32 or ord(c) == 127 for c in value), diagnostic)
+    return value
+
+
+def parse_bootstrap_env(text: str) -> Dict[str, str]:
+    _require(type(text) is str, 'bootstrap.env must contain literal assignments')
+    result = {}
+    for line in text.splitlines():
+        if not line.strip() or line.lstrip().startswith('#'):
+            continue
+        match = _ASSIGNMENT.fullmatch(line)
+        _require(match is not None, 'bootstrap.env must contain literal assignments')
+        key = match.group(1)
+        _require(key in _ENV_KEYS, 'bootstrap.env contains an unsupported assignment')
+        _require(key not in result, 'bootstrap.env contains a duplicate assignment')
+        value = next(part for part in match.groups()[1:] if part is not None)
+        _require(not any(c in value for c in '$`;|&<>'),
+                 'bootstrap.env contains unsupported shell syntax')
+        result[key] = _plain(value, 'bootstrap.env contains an invalid literal', True)
+    return result
+
+
+def _repository(values: Mapping[str, str]) -> Repository:
+    name = values.get('APEX_REPO_NAME', '')
+    _require(bool(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*', name))
+             and name not in ('.', '..'), 'repository name must be one safe directory name')
+    url = values.get('APEX_REPO_URL', '')
+    _require(bool(url) and not any(c.isspace() for c in url), 'repository URL is invalid')
+    if '://' in url:
+        try:
+            parsed = urlsplit(url)
+            port = parsed.port
+        except ValueError:
+            raise ContractError('repository URL is invalid') from None
+        _require(parsed.scheme in ('https', 'ssh') and bool(parsed.hostname)
+                 and parsed.password is None and not parsed.query and not parsed.fragment
+                 and bool(parsed.path.strip('/')) and (port is None or port > 0),
+                 'repository URL is invalid or contains credentials')
+        _require(parsed.scheme != 'https' or parsed.username is None,
+                 'HTTPS repository URL must not contain credentials')
+    else:
+        _require(bool(re.fullmatch(
+            r'[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:[A-Za-z0-9_./-]+', url)),
+            'repository URL must use HTTPS or SSH')
+        remote_path = url.split(':', 1)[1]
+        _require(not remote_path.startswith('-') and remote_path not in ('.', '..'),
+                 'repository URL is invalid')
+    ref = values.get('APEX_REPO_REF') or None
+    if ref is not None:
+        _require(not ref.startswith('-') and not any(c.isspace() for c in ref),
+                 'repository ref is invalid')
+    return Repository(name, url, ref)
+
+
+def _tiers(values: Mapping[str, str]) -> Tuple[Tier, ...]:
+    result = []
+    devices = set()
+    guids = set()
+    for number in (1, 2, 3):
+        prefix = f'APEX_TIER{number}_'
+        device = values.get(prefix + 'DEVICE') or None
+        mode = values.get(prefix + 'MODE')
+        guid = values.get(prefix + 'POOL_GUID') or None
+        _require(mode is not None or device is None,
+                 'a mapped tier device requires an explicit mode')
+        mode = mode or 'directory'
+        _require(mode in ('directory', 'create', 'import'), 'tier mode is invalid')
+        if mode == 'directory':
+            _require(device is None and guid is None,
+                     'directory tier must not select a device or pool')
+        else:
+            _require(device is not None and device.startswith('/dev/')
+                     and not any(c.isspace() for c in device)
+                     and '..' not in PurePosixPath(device).parts
+                     and str(PurePosixPath(device)) == device,
+                     'block-backed tier requires a normalized device path')
+            _require(device not in devices, 'a device is selected by multiple tiers')
+            devices.add(device)
+            if mode == 'create':
+                _require(guid is None, 'create tier must not name an existing pool')
+            else:
+                _require(guid is not None and bool(re.fullmatch(r'[1-9][0-9]{0,19}', guid))
+                         and int(guid) < 2**64, 'import tier requires a valid pool GUID')
+                _require(guid not in guids, 'a pool is selected by multiple tiers')
+                guids.add(guid)
+        result.append(Tier(number, mode, device, guid))
+    return tuple(result)
+
+
+def normalize_user_data(document: Any) -> Enrollment:
+    doc = _mapping(document,
+                   {'preserve_hostname', 'hostname', 'users', 'write_files', 'runcmd'},
+                   set(), 'unsupported cloud-config structure')
+    _require(doc['preserve_hostname'] is False, 'preserve_hostname must be false')
+    hostname = _plain(doc['hostname'], 'hostname is invalid')
+    labels = hostname.split('.')
+    _require(len(hostname) <= 253 and len(labels) >= 2
+             and all(re.fullmatch(r'[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?', x)
+                     for x in labels), 'hostname must be a valid FQDN')
+    users = doc['users']
+    _require(type(users) is list and len(users) == 1, 'exactly one administrator is required')
+    user = _mapping(users[0],
+                    {'name', 'sudo', 'groups', 'shell', 'ssh_authorized_keys'},
+                    set(), 'unsupported administrator configuration')
+    _require(user['name'] == 'adam' and user['shell'] == '/bin/bash',
+             'administrator must be adam with Bash')
+    _require(type(user['sudo']) is str and ''.join(user['sudo'].split()) in
+             ('ALL=(ALL)NOPASSWD:ALL', 'ALL=(ALL:ALL)NOPASSWD:ALL'),
+             'administrator requires passwordless sudo')
+    groups = user['groups']
+    _require(type(groups) is list and len(groups) == 2
+             and all(type(g) is str for g in groups) and set(groups) == {'sudo', 'docker'},
+             'administrator requires sudo and docker groups')
+    keys = user['ssh_authorized_keys']
+    _require(type(keys) is list and len(keys) > 0, 'administrator requires public SSH keys')
+    authorized_keys = tuple(_plain(k, 'administrator SSH key is invalid') for k in keys)
+    _require(all(k.strip() for k in authorized_keys), 'administrator SSH key is empty')
+    commands = doc['runcmd']
+    _require(commands in (['/usr/local/bin/apex-bootstrap'],
+                          [['/usr/local/bin/apex-bootstrap']]),
+             'only the standard APEX bootstrap command is supported')
+    writes = doc['write_files']
+    _require(type(writes) is list, 'write_files must be a list')
+    files = []
+    seen = set()
+    for write in writes:
+        item = _mapping(write, {'path', 'owner', 'permissions', 'content'}, {'defer'},
+                        'unsupported write_files entry')
+        path = _plain(item['path'], 'managed file path is invalid')
+        _require(path in _FILE_POLICY and path not in seen,
+                 'managed file path is unsupported or repeated')
+        seen.add(path)
+        owner, mode, deferred = _FILE_POLICY[path]
+        permission = item['permissions']
+        _require(type(permission) is str and bool(re.fullmatch(r'0?[0-7]{3}', permission))
+                 and int(permission, 8) == mode and item['owner'] == owner,
+                 'managed file ownership or permissions are invalid')
+        _require(item.get('defer', False) is deferred, 'managed file defer setting is invalid')
+        content = item['content']
+        _require(type(content) is str and bool(content.strip()) and '\x00' not in content,
+                 'managed file content is invalid')
+        files.append(ManagedFile(path, owner, mode, content))
+    bootstrap = next((f for f in files if f.path == '/etc/apex/bootstrap.env'), None)
+    _require(bootstrap is not None, 'bootstrap.env is required')
+    values = parse_bootstrap_env(bootstrap.content)
+    return Enrollment(hostname, _repository(values), _tiers(values), authorized_keys, tuple(files))

ci/scripts/init-node/provisioning/image.py

@@ -0,0 +1,124 @@
+"""Image-only installation and sanitization; never called by remote enrollment."""
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import os
+from pathlib import Path
+import shutil
+import tarfile
+import uuid
+
+from . import ENROLLMENT_INTERFACE
+from .state import ProvisionError, atomic_write, exclusive_lock, safe_path
+from .target import preflight, run
+
+
+def install():
+    from .transport import build_bundle
+    source = Path(__file__).resolve().parents[2]
+    data, digest, _ = build_bundle(source)
+    destination = safe_path(Path('/usr/local/lib/apex-provisioner') / digest)
+    destination.mkdir(mode=0o755, parents=True, exist_ok=False)
+    with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+        archive.extractall(destination, filter='data')
+    revision = os.environ.get('APEX_PROVISION_REVISION', 'unversioned-modified')
+    metadata = {'bundle_digest': digest, 'revision': revision, 'path': str(destination),
+                'baseline_revision': revision, 'enrollment_interface': ENROLLMENT_INTERFACE}
+    atomic_write('/etc/apex/provisioner.json', json.dumps(metadata) + '\n')
+    for executable, operation in (('apex-provision-image', ''), ('apex-bootstrap', 'cloud-enroll ')):
+        content = ('#!/bin/sh\nset -eu\ncd ' + str(destination) + '\n'
+                   'exec /usr/bin/python3 -m engine.provisioning.image ' + operation + '"$@"\n')
+        atomic_write('/usr/local/bin/' + executable, content, 0o755)
+    return metadata
+
+
+def cloud_enroll():
+    from .access import Transition
+    from .target import enroll
+    from .userdata import decode_user_data
+    metadata = json.loads(Path('/etc/apex/provisioner.json').read_text())
+    if metadata.get('enrollment_interface') != ENROLLMENT_INTERFACE:
+        raise ProvisionError('image enrollment interface is missing or unsupported; rebuild the image')
+    instance = Path('/var/lib/cloud/instance').resolve(strict=True)
+    try:
+        instance.relative_to('/var/lib/cloud/instances')
+    except ValueError:
+        raise RuntimeError('cloud-init instance path is outside its managed directory') from None
+    path = safe_path(instance / 'user-data.txt')
+    if path.stat().st_uid != 0 or path.stat().st_size > 1024 * 1024:
+        raise RuntimeError('cloud-init user-data is not a supported root-owned file')
+    spec = decode_user_data(path.read_bytes())
+    result = enroll(spec, metadata['revision'], metadata['bundle_digest'])
+    token = uuid.uuid4().hex
+    transition = Transition()
+    try:
+        current = transition.execute('begin', token, 22, '127.0.0.1')
+        run(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true'])
+        if current['status'] not in ('existing', 'committed'):
+            transition.execute('finalize', token)
+        if current['status'] != 'committed':
+            transition.execute('commit', token)
+    except Exception:
+        transition.execute('rollback', token)
+        raise
+    result['external_access'] = 'not-verified-cloud-init'
+    atomic_write('/var/lib/apex-init/enrollment.json', json.dumps(result, sort_keys=True) + '\n')
+    return result
+
+
+def sanitize_files(root=Path('/')):
+    root = Path(root)
+    for home in ('root', 'home/debian'):
+        for name in ('.ssh/authorized_keys', '.ssh/authorized_keys2', '.bash_history'):
+            path = root / home / name
+            if path.exists() or path.is_symlink():
+                path.unlink()
+    for path in (root / 'etc/ssh').glob('ssh_host_*'):
+        if path.is_file() or path.is_symlink():
+            path.unlink()
+    atomic_write(root / 'etc/machine-id', '', 0o444)
+    dbus = root / 'var/lib/dbus/machine-id'
+    if dbus.exists() or dbus.is_symlink():
+        dbus.unlink()
+    dbus.parent.mkdir(parents=True, exist_ok=True)
+    dbus.symlink_to('/etc/machine-id')
+    for name in ('var/lib/apex-init', 'tmp/engine', 'var/lib/cloud'):
+        path = root / name
+        if path.is_symlink():
+            path.unlink()
+        elif path.exists():
+            shutil.rmtree(path)
+    autologin = root / 'etc/systemd/system/serial-getty@ttyS0.service.d/autologin.conf'
+    autologin.unlink(missing_ok=True)
+    atomic_write(root / 'etc/systemd/system/ssh.service.d/10-generate-host-keys.conf',
+                 '[Service]\nExecStartPre=\nExecStartPre=/usr/bin/ssh-keygen -A\n'
+                 'ExecStartPre=/usr/sbin/sshd -t\n', 0o644)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('operation', choices=('install', 'baseline', 'install-runner', 'sanitize', 'cloud-enroll'))
+    args = parser.parse_args(argv)
+    preflight()
+    if args.operation == 'install':
+        print(json.dumps(install()))
+    elif args.operation == 'baseline':
+        from .baseline import apply_baseline
+        with exclusive_lock('/var/lib/apex-init/init.lock'):
+            result = apply_baseline()
+            atomic_write('/etc/apex/image-baseline.json', json.dumps(result, sort_keys=True) + '\n')
+    elif args.operation == 'install-runner':
+        run(['test', '-x', '/usr/local/bin/apex-bootstrap'])
+    elif args.operation == 'sanitize':
+        run(['usermod', '--password', '!', 'root'])
+        run(['usermod', '--password', '!', 'debian'])
+        sanitize_files()
+    else:
+        print(json.dumps(cloud_enroll(), sort_keys=True))
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

ci/scripts/init-node/provisioning/repository.py

@@ -0,0 +1,121 @@
+"""Pinned, non-destructive checkout enrollment as the node administrator."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import pwd
+import subprocess
+
+from engine.identity import read_env
+from . import ENROLLMENT_INTERFACE
+from .state import ProvisionError, safe_path
+
+
+def as_adam(args, cwd=None):
+    environment = dict(os.environ, GIT_TERMINAL_PROMPT='0',
+                       GIT_SSH_COMMAND='ssh -o BatchMode=yes -o StrictHostKeyChecking=yes',
+                       LC_ALL='C')
+    result = subprocess.run(['runuser', '-u', 'adam', '--', *args], cwd=cwd,
+                            env=environment, text=True, capture_output=True, timeout=300)
+    if result.returncode:
+        # Git/hook stderr may contain credential files or remote URL credentials.
+        raise ProvisionError('repository operation failed; check administrator Git access and the pinned checkout')
+    return result.stdout.strip()
+
+
+def verify_checkout_setup(destination, account):
+    if as_adam(['git', 'config', '--get', 'core.hooksPath'], destination) != 'commons/githooks':
+        raise ProvisionError('checkout hook configuration is missing or incompatible')
+    home = safe_path(account.pw_dir)
+    bashrc = safe_path(home / '.bashrc')
+    entry = 'export PATH="' + str(destination / 'commons') + ':$PATH"'
+    if not bashrc.is_file() or bashrc.read_text().splitlines().count(entry) != 1:
+        raise ProvisionError('checkout PATH setup is missing or duplicated; reconcile init.sh before retrying')
+    vim = home / '.vimrc'
+    if not vim.is_symlink() or vim.resolve() != (destination / 'commons/etc/.vimrc').resolve():
+        raise ProvisionError('checkout editor link is missing or incompatible')
+
+
+def reconcile_repository(spec, state, save, srv=Path('/srv')):
+    repository = spec.repository
+    destination = safe_path(Path(srv) / repository.name)
+    staging = safe_path(Path(srv) / ('.apex-clone-' + repository.name))
+    expected = {'name': repository.name, 'url': repository.url, 'ref': repository.ref}
+    account = pwd.getpwnam('adam')
+    record = state.setdefault('repository', {})
+    if record and record.get('request') != expected:
+        raise ProvisionError('repository enrollment conflicts with its recorded request')
+    if not record:
+        if destination.exists() or staging.exists():
+            raise ProvisionError('unrecorded repository path exists; explicit adoption is required')
+        record.update(request=expected, clone_intent=True)
+        save()
+    if not destination.exists():
+        if staging.exists():
+            # An interrupted clone may be incomplete: never erase or guess its state.
+            raise ProvisionError('an interrupted clone remains; inspect the staging checkout before retrying')
+        staging.mkdir(mode=0o755)
+        os.chown(staging, account.pw_uid, account.pw_gid)
+        as_adam(['git', 'clone', '--no-checkout', '--', repository.url, str(staging)])
+        os.rename(staging, destination)
+    safe_path(destination / '.git')
+    origin = as_adam(['git', 'remote', 'get-url', 'origin'], destination)
+    if origin != repository.url:
+        raise ProvisionError('existing checkout has a different origin; it was preserved')
+    if 'commit' not in record:
+        reference = repository.ref or 'HEAD'
+        if repository.ref:
+            as_adam(['git', 'fetch', '--no-tags', 'origin', reference], destination)
+            reference = 'FETCH_HEAD'
+        commit = as_adam(['git', 'rev-parse', '--verify', reference + '^{commit}'], destination)
+        record['commit'] = commit
+        record['checkout_intent'] = True
+        save()
+    commit = record['commit']
+    if record.get('checkout_intent'):
+        # --no-checkout index intentionally looks dirty before the first checkout.
+        if any(p.name != '.git' for p in destination.iterdir()):
+            head = as_adam(['git', 'rev-parse', 'HEAD'], destination)
+            dirty = as_adam(['git', 'status', '--porcelain'], destination)
+            if head != commit or dirty:
+                raise ProvisionError('interrupted checkout has unexpected contents; it was preserved')
+        else:
+            as_adam(['git', 'checkout', '--detach', commit], destination)
+        record.pop('checkout_intent')
+        save()
+    if as_adam(['git', 'rev-parse', 'HEAD'], destination) != commit:
+        raise ProvisionError('checkout revision changed; initialization will not reset it')
+    if as_adam(['git', 'status', '--porcelain'], destination):
+        raise ProvisionError('checkout contains local changes; initialization will not reset it')
+    node_env = safe_path(destination / 'node.env')
+    node_identity = read_env(str(node_env))
+    if node_identity.get('APEX_NODE_FQDN') != spec.hostname:
+        raise ProvisionError('repository node.env does not declare the requested hostname')
+    if node_identity.get('APEX_ENROLLMENT_INTERFACE') != str(ENROLLMENT_INTERFACE):
+        raise ProvisionError('node checkout enrollment interface is missing or unsupported; update its init.sh and node.env together')
+    as_adam(['git', 'submodule', 'update', '--init', '--recursive'], destination)
+    commons = safe_path(destination / 'commons')
+    if not commons.is_dir():
+        raise ProvisionError('node checkout does not contain its pinned commons submodule')
+    commons_commit = as_adam(['git', 'rev-parse', 'HEAD'], commons)
+    gitlink = as_adam(['git', 'ls-tree', 'HEAD', '--', 'commons'], destination).split()
+    if gitlink != ['160000', 'commit', commons_commit, 'commons']:
+        raise ProvisionError('commons is not the exact pinned Git submodule')
+    if Path(as_adam(['git', 'rev-parse', '--show-toplevel'], commons)).resolve() != commons.resolve():
+        raise ProvisionError('commons does not have its own Git worktree')
+    if record.get('commons_commit', commons_commit) != commons_commit:
+        raise ProvisionError('pinned commons revision changed unexpectedly')
+    hook = safe_path(destination / 'init.sh')
+    if not hook.is_file():
+        raise ProvisionError('node repository is missing its checkout initializer')
+    if not record.get('hook_complete'):
+        as_adam(['bash', './init.sh'], destination)
+        verify_checkout_setup(destination, account)
+        record['hook_complete'] = True
+    else:
+        verify_checkout_setup(destination, account)
+    if as_adam(['git', 'status', '--porcelain'], destination):
+        raise ProvisionError('checkout initializer changed tracked files; inspect before resuming')
+    record['commons_commit'] = commons_commit
+    save()
+    return {'path': str(destination), 'commit': commit, 'commons_commit': commons_commit}

ci/scripts/init-node/provisioning/security.py

@@ -0,0 +1,591 @@
+"""Security activation owned by the protected access transaction."""
+from __future__ import annotations
+
+import base64
+import hashlib
+import ipaddress
+import json
+import os
+import re
+import shlex
+import tempfile
+import uuid
+
+from .access import atomic
+from .baseline import HELPER_SHA256
+
+
+SERVICES = ('ufw', 'crowdsec', 'crowdsec-firewall-bouncer')
+FILES = ('/etc/default/ufw', '/etc/ufw/ufw.conf', '/etc/ufw/before.rules',
+         '/etc/ufw/before6.rules', '/etc/ufw/after.rules', '/etc/ufw/after6.rules',
+         '/etc/ufw/user.rules', '/etc/ufw/user6.rules')
+HELPER = '/usr/local/bin/ufw-docker'
+BLOCK = re.compile(r'^# BEGIN UFW AND DOCKER\n.*?^# END UFW AND DOCKER\n?', re.M | re.S)
+
+
+def _helper_lines(ipv6):
+    prefix = 'ufw6' if ipv6 else 'ufw'
+    subnets = ('fd00::/8',) if ipv6 else ('10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16')
+    return ['# BEGIN UFW AND DOCKER', '*filter', ':' + prefix + '-user-forward - [0:0]',
+            ':' + prefix + '-docker-logging-deny - [0:0]', ':DOCKER-USER - [0:0]',
+            '-A DOCKER-USER -j ' + prefix + '-user-forward',
+            '-A DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j RETURN',
+            '-A DOCKER-USER -m conntrack --ctstate INVALID -j DROP',
+            '-A DOCKER-USER -i docker0 -o docker0 -j ACCEPT',
+            *['-A DOCKER-USER -j RETURN -s ' + subnet for subnet in subnets],
+            *['-A DOCKER-USER -j ' + prefix + '-docker-logging-deny -m conntrack --ctstate NEW -d ' + subnet
+              for subnet in subnets],
+            '-A DOCKER-USER -j RETURN',
+            '-A ' + prefix + '-docker-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW DOCKER BLOCK] "',
+            '-A ' + prefix + '-docker-logging-deny -j DROP', 'COMMIT', '# END UFW AND DOCKER']
+
+
+def _runtime_rule(line):
+    tokens = shlex.split(line)
+    if len(tokens) % 2:
+        raise SecurityError('Unsupported runtime firewall rule')
+    pairs = []
+    for key, value in zip(tokens[::2], tokens[1::2]):
+        if key in ('-s', '-d'):
+            value = str(ipaddress.ip_network(value, strict=False))
+        elif key == '--ctstate':
+            value = ','.join(sorted(value.split(',')))
+        pairs.append((key, value))
+    return sorted(pairs)
+
+
+class SecurityError(RuntimeError):
+    pass
+
+
+class Security:
+    def __init__(self, host):
+        self.host = host
+
+    def preflight(self):
+        services = {name: self._service(name) for name in SERVICES}
+        for name, state in services.items():
+            if (state.get('LoadState') != 'loaded' or state.get('UnitFileState') not in ('enabled', 'disabled')
+                    or state.get('ActiveState') not in ('active', 'inactive')
+                    or state.get('DropInPaths') or not self._supported_unit(name, state.get('FragmentPath'))):
+                raise SecurityError('Unsupported or masked security service: ' + name)
+        if self._digest(self._path(HELPER).read_bytes()) != HELPER_SHA256:
+            raise SecurityError('The ufw-docker helper does not match the pinned baseline')
+        files = self._files()
+        active = self._active()
+        pristine = False
+        if not active:
+            output = self.host.command(['dpkg-query', '-W', '-f=${Conffiles}', 'ufw']).stdout
+            conffiles = dict(re.findall(r'^\s*(/\S+)\s+([a-f0-9]{32})(?:\s|$)', output, re.M))
+            pristine = (hashlib.md5(base64.b64decode(files['/etc/default/ufw']['data'])).hexdigest()
+                        == conffiles.get('/etc/default/ufw'))
+            manifest = dict((name, digest) for digest, name in re.findall(
+                r'^([a-f0-9]{32})\s+(\S+)\s*$', self._text('/var/lib/dpkg/info/ufw.md5sums'), re.M))
+            # Debian's postinst copies/ucf-installs these files; they are not dpkg conffiles.
+            for name in FILES[1:]:
+                filename = name.rsplit('/', 1)[1]
+                template = '/usr/share/ufw/' + ('' if filename == 'ufw.conf' else 'iptables/') + filename
+                data = self._path(template).read_bytes()
+                pristine = pristine and (hashlib.md5(data).hexdigest() == manifest.get(template.lstrip('/'))
+                                         and data == base64.b64decode(files[name]['data']))
+            if not pristine:
+                raise SecurityError('Inactive UFW is not provably pristine; reconcile its existing policy first')
+        texts = [self._text('/etc/ufw/' + name) for name in ('after.rules', 'after6.rules')]
+        present = any('UFW AND DOCKER' in text for text in texts)
+        if present:
+            self._check_helper_files()
+            runtime = None
+        else:
+            runtime = {executable: self._helper_runtime(executable) for executable in ('iptables', 'ip6tables')}
+            for chains in runtime.values():
+                self._validate_helper_before(chains)
+        return {'version': 1, 'pristine': pristine, 'active': active, 'services': services,
+                'files': files, 'helper_present': present, 'helper_runtime_before': runtime,
+                'ipv6': bool(re.search(r'^IPV6\s*=\s*[\"\x27]?yes', self._text('/etc/default/ufw'), re.M))}
+
+    def _supported_unit(self, name, path):
+        if path in ('/usr/lib/systemd/system/' + name + '.service', '/lib/systemd/system/' + name + '.service'):
+            return True
+        if name != 'crowdsec-firewall-bouncer' or path != '/etc/systemd/system/crowdsec-firewall-bouncer.service':
+            return False
+        output = self.host.command(['dpkg-query', '-W', '-f=${Conffiles}',
+                                    'crowdsec-firewall-bouncer-iptables']).stdout
+        conffiles = dict(re.findall(r'^\s*(/\S+)\s+([a-f0-9]{32})(?:\s|$)', output, re.M))
+        return hashlib.md5(self._path(path).read_bytes()).hexdigest() == conffiles.get(path)
+
+    def _path(self, name):
+        path = self.host.path(name)
+        for parent in (path, *path.parents):
+            if parent.is_symlink():
+                raise SecurityError('Refusing redirected security path: ' + name)
+        if not path.is_file():
+            raise SecurityError('Required security file is missing: ' + name)
+        return path
+
+    def _text(self, name):
+        return self._path(name).read_text()
+
+    @staticmethod
+    def _digest(data):
+        return hashlib.sha256(data).hexdigest()
+
+    def _files(self):
+        return {name: self._file(name) for name in FILES}
+
+    def _file(self, name):
+        path = self._path(name)
+        data, metadata = path.read_bytes(), path.stat()
+        result = {'data': base64.b64encode(data).decode(), 'digest': self._digest(data),
+                  'mode': metadata.st_mode & 0o777, 'uid': metadata.st_uid, 'gid': metadata.st_gid}
+        result['fingerprint'] = self._digest(json.dumps(result, sort_keys=True).encode())
+        return result
+
+    def _service(self, name):
+        output = self.host.command(['systemctl', 'show', name, '-p',
+                                   'LoadState,ActiveState,UnitFileState,FragmentPath,DropInPaths']).stdout
+        return dict(line.split('=', 1) for line in output.splitlines() if '=' in line)
+
+    def _active(self):
+        output = self.host.command(['ufw', 'status']).stdout
+        if 'Status: inactive' in output:
+            return False
+        if 'Status: active' in output:
+            return True
+        raise SecurityError('Cannot determine UFW runtime state')
+
+    def snapshot(self, plan, original_port, client_address):
+        if not isinstance(original_port, int) or not 1 <= original_port <= 65535:
+            raise SecurityError('Invalid bootstrap port')
+        try:
+            address = str(ipaddress.ip_address(client_address))
+        except ValueError:
+            raise SecurityError('Invalid bootstrap peer address') from None
+        if plan != self.preflight():
+            raise SecurityError('Security configuration changed after preflight')
+        if ipaddress.ip_address(address).version == 6 and not plan['ipv6']:
+            raise SecurityError('IPv6 bootstrap access requires enabled UFW IPv6 support')
+        if plan['services']['crowdsec']['ActiveState'] == 'active':
+            self._check_decisions(address)
+        return {'version': 1, 'plan': plan, 'original_port': original_port,
+                'client_address': address, 'token': uuid.uuid4().hex, 'rules': [],
+                'expected': {name: value['fingerprint'] for name, value in plan['files'].items()},
+                'service_changes': [], 'inflight': None, 'phase': 'prepared',
+                'access_recovered': False, 'cleanup': 'not-started', 'helper_blocks': {},
+                'helper_runtime': {'introduced': False, 'restored': []}}
+
+    def activate(self, state, persist):
+        if state['phase'] != 'prepared' or state['inflight']:
+            raise SecurityError('Security activation requires a fresh prepared snapshot')
+        self._assert_files(state)
+        state['phase'] = 'activating'
+        persist(state)
+        self._add_rule(state, state['original_port'], state['client_address'], True, persist)
+        self._add_rule(state, 2222, '0.0.0.0/0', False, persist)
+        if state['plan']['ipv6']:
+            self._add_rule(state, 2222, '::/0', False, persist)
+        if state['plan']['pristine']:
+            for policy, direction in (('deny', 'incoming'), ('allow', 'outgoing'), ('allow', 'routed')):
+                self._mutate(state, ['ufw', 'default', policy, direction], ['/etc/default/ufw'], persist)
+        self._mutate(state, ['ufw', '--force', 'enable'],
+                     ['/etc/ufw/ufw.conf', '/etc/ufw/user.rules', '/etc/ufw/user6.rules'], persist)
+        if not state['plan']['helper_present']:
+            if any(self._helper_runtime(executable) != before for executable, before in
+                   state['plan']['helper_runtime_before'].items()):
+                raise SecurityError('Docker firewall rules changed before helper installation')
+            state['helper_runtime']['introduced'] = True
+            persist(state)
+            names = ['/etc/ufw/after.rules', '/etc/ufw/after6.rules']
+            self._mutate(state, [HELPER, 'install'], names, persist)
+            for name in names:
+                blocks = BLOCK.findall(self._text(name))
+                if len(blocks) != 1:
+                    raise SecurityError('ufw-docker did not install a verifiable integration block')
+                state['helper_blocks'][name] = blocks[0]
+            persist(state)
+        self._check_helper_files()
+        self._mutate(state, ['ufw', 'reload'], [], persist)
+        self._enable(state, 'ufw', persist)
+        self.host.command(['crowdsec', '-t', '-c', '/etc/crowdsec/config.yaml'])
+        self._enable(state, 'crowdsec', persist)
+        self.host.command(['cscli', 'lapi', 'status'])
+        self._check_decisions(state['client_address'])
+        self._enable(state, 'crowdsec-firewall-bouncer', persist)
+        state['phase'] = 'active'
+        persist(state)
+        self.verify(state)
+
+    def finalize(self, state, persist):
+        self.verify(state)
+        for rule in state['rules']:
+            if rule['temporary'] and rule['owned']:
+                self._delete_rule(state, rule, persist)
+        state['phase'] = 'final'
+        persist(state)
+        self.verify(state)
+
+    def verify(self, state):
+        if state['phase'] not in ('active', 'final') or state['inflight']:
+            raise SecurityError('Security activation is incomplete')
+        self._assert_files(state)
+        if not self._active():
+            raise SecurityError('UFW is not active')
+        for name in SERVICES:
+            service = self._service(name)
+            if service.get('ActiveState') != 'active' or service.get('UnitFileState') != 'enabled':
+                raise SecurityError('Security service is not active and enabled: ' + name)
+        self.host.command(['cscli', 'lapi', 'status'])
+        self._check_decisions(state['client_address'])
+        self._check_helper_files()
+        self._check_helper_runtime(state['plan']['ipv6'])
+        for rule in state['rules']:
+            if not (rule['temporary'] and state['phase'] == 'final') and self._rule_comment(rule) is None:
+                raise SecurityError('Required SSH firewall allowance is missing')
+
+    def _check_helper_files(self):
+        for ipv6, name in ((False, 'after.rules'), (True, 'after6.rules')):
+            text = self._text('/etc/ufw/' + name)
+            blocks = BLOCK.findall(text)
+            if (len(blocks) != 1 or text.count('UFW AND DOCKER') != 2
+                    or [line.strip() for line in blocks[0].splitlines() if line.strip()] != _helper_lines(ipv6)):
+                raise SecurityError('Existing ufw-docker integration is custom or incomplete')
+
+    def _check_helper_runtime(self, ipv6):
+        for version in (False, True) if ipv6 else (False,):
+            executable = 'ip6tables' if version else 'iptables'
+            prefix = 'ufw6' if version else 'ufw'
+            for chain in ('DOCKER-USER', prefix + '-docker-logging-deny'):
+                result = self.host.command([executable, '-S', chain], False)
+                expected = ['-N ' + chain] + [line for line in _helper_lines(version)
+                                              if line.startswith('-A ' + chain + ' ')]
+                try:
+                    matches = (not result.returncode and [_runtime_rule(line) for line in result.stdout.splitlines()]
+                               == [_runtime_rule(line) for line in expected])
+                except ValueError:
+                    matches = False
+                if not matches:
+                    raise SecurityError('ufw-docker runtime chain is missing or changed: ' + executable + ' ' + chain)
+            if self.host.command([executable, '-C', 'FORWARD', '-j', 'DOCKER-USER'], False).returncode:
+                raise SecurityError('Docker forwarding hook is missing: ' + executable)
+
+    def _helper_runtime(self, executable):
+        logging = ('ufw6' if executable == 'ip6tables' else 'ufw') + '-docker-logging-deny'
+        chains = {'DOCKER-USER': None, logging: None}
+        for line in self.host.command([executable, '-S']).stdout.splitlines():
+            tokens = shlex.split(line)
+            if len(tokens) < 2 or tokens[1] not in chains:
+                continue
+            if tokens[0] == '-N' and len(tokens) == 2 and chains[tokens[1]] is None:
+                chains[tokens[1]] = [line]
+            elif tokens[0] == '-A' and chains[tokens[1]] is not None:
+                chains[tokens[1]].append(line)
+            else:
+                raise SecurityError('Unsupported Docker firewall chain declaration')
+        return chains
+
+    @staticmethod
+    def _validate_helper_before(chains):
+        if (not isinstance(chains, dict) or len(chains) != 2 or 'DOCKER-USER' not in chains
+                or chains['DOCKER-USER'] not in (None, ['-N DOCKER-USER'],
+                                                ['-N DOCKER-USER', '-A DOCKER-USER -j RETURN'])
+                or any(value is not None for name, value in chains.items() if name != 'DOCKER-USER')):
+            raise SecurityError('Unmanaged Docker firewall rules require reconciliation')
+
+    def _restore_helper_runtime(self, state, persist):
+        if state['plan']['helper_present']:
+            return
+        before = state['plan'].get('helper_runtime_before')
+        progress = state.get('helper_runtime')
+        if (not isinstance(before, dict) or set(before) != {'iptables', 'ip6tables'}
+                or not isinstance(progress, dict) or not isinstance(progress.get('introduced'), bool)
+                or not isinstance(progress.get('restored'), list)):
+            raise SecurityError('Missing Docker firewall recovery snapshot; reconcile explicitly')
+        for executable, original in before.items():
+            self._validate_helper_before(original)
+            current = self._helper_runtime(executable)
+            if set(original) != set(current):
+                raise SecurityError('Unsupported Docker firewall recovery snapshot')
+            if current != original:
+                ipv6 = executable == 'ip6tables'
+                expected = {name: ['-N ' + name] + [line for line in _helper_lines(ipv6)
+                            if line.startswith('-A ' + name + ' ')] for name in current}
+                try:
+                    owned = (progress['introduced'] and (not ipv6 or state['plan']['ipv6'])
+                             and all(current[name] is not None and
+                                     [_runtime_rule(line) for line in current[name]] ==
+                                     [_runtime_rule(line) for line in expected[name]] for name in current))
+                except (ValueError, SecurityError):
+                    owned = False
+                if not owned:
+                    raise SecurityError('Docker firewall runtime changed; preserve it for reconciliation')
+                logging = ('ufw6' if ipv6 else 'ufw') + '-docker-logging-deny'
+                commands = ['*filter', '-F DOCKER-USER', *(original['DOCKER-USER'] or [])[1:],
+                            '-F ' + logging, '-X ' + logging]
+                if original['DOCKER-USER'] is None:
+                    commands.append('-X DOCKER-USER')
+                commands.append('COMMIT')
+                progress['restoring'] = executable
+                persist(state)
+                with tempfile.NamedTemporaryFile(mode='w', prefix='apex-firewall-', suffix='.rules') as stream:
+                    stream.write('\n'.join(commands) + '\n')
+                    stream.flush()
+                    if self._helper_runtime(executable) != current:
+                        raise SecurityError('Docker firewall runtime changed immediately before restoration')
+                    self.host.command([executable + '-restore', '--wait', '10', '--noflush', stream.name])
+                if self._helper_runtime(executable) != original:
+                    raise SecurityError('Docker firewall restoration did not converge')
+            if executable not in progress['restored']:
+                progress['restored'].append(executable)
+            progress.pop('restoring', None)
+            persist(state)
+
+    def restore(self, state, persist):
+        state['cleanup'] = 'pending'
+        state['access_recovered'] = False
+        persist(state)
+        errors = []
+        for change in reversed(state['service_changes']):
+            name, field = change['service'], change['field']
+            if name == 'ufw' and field == 'ActiveState' and state['plan']['active']:
+                continue
+            try:
+                current = self._service(name)
+                if any(current.get(key) != change['metadata'].get(key) for key in change['metadata']):
+                    raise SecurityError('Service unit changed during activation')
+                owned_start = (field == 'ActiveState' and change['before'] == 'inactive'
+                               and change['after'] == 'active' and current[field] in
+                               ('active', 'inactive', 'activating', 'deactivating', 'failed', 'reloading'))
+                # Stop even when inactive: systemd may still have a queued start job.
+                if current[field] == change['after'] or owned_start:
+                    actions = {'active': 'start', 'inactive': 'stop', 'enabled': 'enable', 'disabled': 'disable'}
+                    change['status'] = 'restoring'
+                    persist(state)
+                    self.host.command(['systemctl', actions[change['before']], name])
+                    stopped = self._service(name)
+                    if any(stopped.get(key) != change['metadata'].get(key) for key in change['metadata']):
+                        raise SecurityError('Service unit changed during restoration')
+                    if owned_start and stopped[field] == 'failed':
+                        self.host.command(['systemctl', 'reset-failed', name])
+                elif current[field] != change['before']:
+                    raise SecurityError('Service state changed outside this transition')
+                if self._service(name)[field] != change['before']:
+                    raise SecurityError('Service state restoration did not converge')
+                change['status'] = 'restored'
+                persist(state)
+            except Exception:
+                errors.append('service:' + name + ':' + field)
+        ambiguous = []
+        for name in FILES:
+            try:
+                if self._file(name)['fingerprint'] not in (state['plan']['files'][name]['fingerprint'], state['expected'][name]):
+                    ambiguous.append(name)
+            except Exception:
+                ambiguous.append(name)
+        pending = bool(ambiguous or state['inflight'] or errors)
+        original_active = state['plan']['active']
+        # Recover runtime access even when an unrelated edit prevents file cleanup.
+        try:
+            if not original_active:
+                try:
+                    self._path('/etc/ufw/ufw.conf')
+                except SecurityError:
+                    # A service stop removes runtime rules without following the replaced enable file.
+                    self.host.command(['systemctl', 'stop', 'ufw'])
+                else:
+                    self.host.command(['ufw', 'disable'])
+                state['access_recovered'] = not self._active()
+            elif pending:
+                self._retain_bootstrap(state, persist)
+                if not self._active():
+                    self.host.command(['ufw', '--force', 'enable'])
+                state['access_recovered'] = self._active()
+        except Exception:
+            errors.append('bootstrap-firewall')
+        for name, previous in state['plan']['files'].items():
+            try:
+                path = self._path(name)
+                content = path.read_bytes()
+                fingerprint = self._file(name)['fingerprint']
+                if fingerprint == previous['fingerprint']:
+                    continue
+                if fingerprint == state['expected'][name] and not (original_active and pending and name.endswith(('user.rules', 'user6.rules'))):
+                    atomic(path, base64.b64decode(previous['data']), previous['mode'])
+                    os.chown(path, previous['uid'], previous['gid'])
+                    continue
+                block = state['helper_blocks'].get(name)
+                if block and content.decode().count(block) == 1:
+                    metadata = path.stat()
+                    atomic(path, content.replace(block.encode(), b'', 1), metadata.st_mode & 0o777)
+                    os.chown(path, metadata.st_uid, metadata.st_gid)
+                else:
+                    errors.append('file:' + name)
+            except Exception:
+                errors.append('file:' + name)
+        # Rule removal is ownership-checked independently of whole-file snapshots.
+        for rule in state['rules']:
+            if rule['owned'] and not (original_active and pending and rule['temporary']):
+                try:
+                    if self._rule_comment(rule) == rule['comment']:
+                        self.host.command(self._rule_command(rule, delete=True))
+                    elif self._rule_comment(rule) is not None:
+                        errors.append('rule-ownership')
+                except Exception:
+                    errors.append('rule-cleanup')
+        try:
+            self._restore_helper_runtime(state, persist)
+        except Exception:
+            errors.append('docker-firewall-runtime')
+        try:
+            if original_active:
+                if errors or pending:
+                    self._retain_bootstrap(state, persist)
+                else:
+                    self.host.command(['ufw', 'reload'])
+                state['access_recovered'] = self._active()
+            else:
+                # Restoring the former ufw.conf must not reactivate the runtime.
+                state['access_recovered'] = not self._active()
+        except Exception:
+            errors.append('bootstrap-firewall')
+        if not state['access_recovered']:
+            errors.append('bootstrap-firewall')
+        if not state['plan']['helper_present']:
+            try:
+                if any(self._helper_runtime(executable) != before for executable, before in
+                       state['plan']['helper_runtime_before'].items()):
+                    raise SecurityError('Docker firewall restoration changed before cleanup completed')
+            except Exception:
+                errors.append('docker-firewall-runtime')
+        for change in state['service_changes']:
+            name, field = change['service'], change['field']
+            if name == 'ufw' and field == 'ActiveState' and original_active:
+                continue
+            try:
+                current = self._service(name)
+                if (current[field] != change['before'] or any(current.get(key) != value
+                                                            for key, value in change['metadata'].items())):
+                    raise SecurityError('Service restoration changed before cleanup completed')
+            except Exception:
+                errors.append('service:' + name + ':' + field)
+        if errors or pending:
+            state['diagnostics'] = sorted(set(errors + ['file:' + name for name in ambiguous]
+                                              + (['interrupted-mutation'] if state['inflight'] else [])))
+            persist(state)
+            raise SecurityError('Security cleanup pending; bootstrap firewall recovery=' + str(state['access_recovered'])
+                                + '; inspect the root-owned recovery record')
+        state.update(cleanup='complete', phase='restored', inflight=None, diagnostics=[])
+        persist(state)
+
+    def _assert_files(self, state):
+        if any(value['fingerprint'] != state['expected'][name] for name, value in self._files().items()):
+            raise SecurityError('Security files changed outside this transition; recovery remains required')
+
+    def _mutate(self, state, args, changed, persist):
+        self._assert_files(state)
+        state['inflight'] = {'command': args[:2], 'files': changed}
+        persist(state)
+        self.host.command(args)
+        current = self._files()
+        for name in changed:
+            state['expected'][name] = current[name]['fingerprint']
+        if any(value['fingerprint'] != state['expected'][name] for name, value in current.items()):
+            raise SecurityError('Concurrent security file change during activation')
+        state['inflight'] = None
+        persist(state)
+
+    def _enable(self, state, name, persist):
+        for field, before, after, operation in (
+                ('UnitFileState', 'disabled', 'enabled', 'enable'),
+                ('ActiveState', 'inactive', 'active', 'start')):
+            current = self._service(name)
+            if current[field] == after:
+                continue
+            if current[field] != before:
+                raise SecurityError('Security service changed before activation: ' + name)
+            metadata = {key: current.get(key) for key in ('LoadState', 'FragmentPath', 'DropInPaths')}
+            if any(value != state['plan']['services'][name].get(key) for key, value in metadata.items()):
+                raise SecurityError('Security service unit changed before activation: ' + name)
+            change = {'service': name, 'field': field, 'before': before, 'after': after,
+                      'metadata': metadata, 'status': 'pending'}
+            state['service_changes'].append(change)
+            persist(state)
+            self.host.command(['systemctl', operation, name])
+            if self._service(name)[field] != after:
+                raise SecurityError('Security service did not activate: ' + name)
+            change['status'] = 'applied'
+            persist(state)
+
+    def _check_decisions(self, peer):
+        self.host.command(['cscli', 'lapi', 'status'])
+        try:
+            alerts = json.loads(self.host.command(
+                ['cscli', 'decisions', 'list', '--all', '--limit', '0', '--no-simu', '-o', 'json']).stdout)
+            if not isinstance(alerts, list):
+                raise ValueError()
+            address = ipaddress.ip_address(peer)
+            for alert in alerts:
+                if not isinstance(alert, dict) or not isinstance(alert.get('decisions'), list):
+                    raise ValueError()
+                for decision in alert['decisions']:
+                    if (not isinstance(decision, dict) or any(not isinstance(decision.get(key), str)
+                            or not decision[key] for key in ('scope', 'value', 'type'))
+                            or decision.get('simulated', False) is not False):
+                        raise ValueError()
+                    if decision['scope'].lower() not in ('ip', 'range'):
+                        continue
+                    network = ipaddress.ip_network(decision['value'], strict=False)
+                    if address in network:
+                        raise SecurityError('An existing CrowdSec decision covers the bootstrap peer')
+        except (ValueError, TypeError, KeyError):
+            raise SecurityError('Cannot verify CrowdSec decisions safely') from None
+
+    @staticmethod
+    def _rule_file(rule):
+        return '/etc/ufw/user6.rules' if ipaddress.ip_network(rule['source'], strict=False).version == 6 else '/etc/ufw/user.rules'
+
+    def _rule_comment(self, rule):
+        source = ipaddress.ip_network(rule['source'], strict=False)
+        any_address = '::/0' if source.version == 6 else '0.0.0.0/0'
+        for line in self._text(self._rule_file(rule)).splitlines():
+            if not line.startswith('### tuple ### '):
+                continue
+            parts = line[len('### tuple ### '):].split()
+            if len(parts) < 7 or parts[:5] != ['allow', 'tcp', str(rule['port']), any_address, 'any'] or parts[6] != 'in':
+                continue
+            try:
+                if ipaddress.ip_network(parts[5], strict=False) != source:
+                    continue
+                comment = next((p[8:] for p in parts[7:] if p.startswith('comment=')), '')
+                return bytes.fromhex(comment).decode() if comment else ''
+            except (ValueError, UnicodeError):
+                raise SecurityError('Cannot prove ownership of an existing SSH firewall rule') from None
+        return None
+
+    @staticmethod
+    def _rule_command(rule, delete=False):
+        command = ['ufw', '--force', 'delete'] if delete else (['ufw', 'prepend'] if rule['temporary'] else ['ufw'])
+        return command + ['allow', 'proto', 'tcp', 'from', rule['source'], 'to', 'any', 'port', str(rule['port']),
+                          'comment', rule['comment']]
+
+    def _add_rule(self, state, port, source, temporary, persist):
+        rule = {'port': port, 'source': source, 'temporary': temporary,
+                'comment': 'apex-init-' + state['token'] + ('-bootstrap' if temporary else '-final')}
+        rule['owned'] = self._rule_comment(rule) is None
+        state['rules'].append(rule)
+        persist(state)
+        if rule['owned']:
+            self._mutate(state, self._rule_command(rule), ['/etc/ufw/user.rules', '/etc/ufw/user6.rules'], persist)
+
+    def _delete_rule(self, state, rule, persist):
+        if self._rule_comment(rule) != rule['comment']:
+            raise SecurityError('Owned SSH rule changed; preserve it for explicit reconciliation')
+        self._mutate(state, self._rule_command(rule, delete=True), ['/etc/ufw/user.rules', '/etc/ufw/user6.rules'], persist)
+
+    def _retain_bootstrap(self, state, persist):
+        rule = next((rule for rule in state['rules'] if rule['temporary']), None)
+        if rule is None:
+            raise SecurityError('No recorded bootstrap allowance exists')
+        if self._rule_comment(rule) is None:
+            rule['owned'] = True
+            state['bootstrap_retained'] = True
+            persist(state)
+            self.host.command(self._rule_command(rule))

ci/scripts/init-node/provisioning/seed.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Only the parser runtime may be installed before semantic input validation.
+set -euo pipefail
+export LC_ALL=C DEBIAN_FRONTEND=noninteractive
+[[ $(id -u) == 0 ]] || { echo 'APEX seed requires root.' >&2; exit 1; }
+[[ -r /etc/os-release ]] || { echo 'APEX seed requires Debian 13 amd64.' >&2; exit 1; }
+. /etc/os-release
+[[ ${ID:-} == debian && ${VERSION_ID:-} == 13 && $(dpkg --print-architecture) == amd64 ]] || {
+    echo 'APEX seed requires Debian 13 amd64.' >&2; exit 1;
+}
+
+policy=/usr/sbin/policy-rc.d
+state=/usr/sbin/.apex-policy-rc.d
+for path in /var /var/lib /var/lib/apex-init; do
+    [[ ! -L $path ]] || { echo 'Refusing redirected initialization state.' >&2; exit 1; }
+done
+mkdir -p -m 0700 /var/lib/apex-init
+[[ $(stat -c %u /var/lib/apex-init) == 0 ]] || {
+    echo 'Initialization state must be root-owned.' >&2; exit 1;
+}
+(( (8#$(stat -c %a /var/lib/apex-init) & 022) == 0 )) || {
+    echo 'Initialization state must not be writable by other users.' >&2; exit 1;
+}
+[[ ! -L /var/lib/apex-init/init.lock ]] || { echo 'Refusing redirected initialization lock.' >&2; exit 1; }
+exec 8>/var/lib/apex-init/init.lock
+flock -n 8 || { echo 'Another APEX initialization is running.' >&2; exit 1; }
+mkdir -p /run/lock
+exec 9>/run/lock/apex-package-policy.lock
+flock -n 9 || { echo 'Another APEX package operation is running.' >&2; exit 1; }
+restore_policy() {
+    [[ -d $state ]] || return 0
+    [[ ! -L $state && $(stat -c %u "$state") == 0 ]] || {
+        echo 'Refusing untrusted service policy recovery state.' >&2; return 1;
+    }
+    if [[ -e $state/ready && ( -e $policy || -L $policy ) ]]; then
+        if ! cmp -s "$policy" <(printf '#!/bin/sh\nexit 101\n') &&
+           ! cmp -s "$policy" "$state/original"; then
+            echo 'Service policy changed outside APEX; reconcile it explicitly.' >&2
+            return 1
+        fi
+    fi
+    if [[ -e $state/original || -L $state/original ]]; then
+        rm -f -- "$state/restoring"
+        cp -a -- "$state/original" "$state/restoring"
+        mv -Tf -- "$state/restoring" "$policy"
+    elif [[ -e $state/absent ]]; then
+        rm -f -- "$policy"
+    elif [[ -e $state/ready ]]; then
+        echo 'APEX service policy recovery is incomplete; inspect /usr/sbin/.apex-policy-rc.d.' >&2
+        return 1
+    fi
+    rm -rf -- "$state"
+}
+
+# Release the shared host lock on exit; the target runner reacquires it later.
+restore_policy
+missing=()
+for package in python3 python3-yaml; do
+    status=$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null || true)
+    [[ $status == *' ok installed' ]] || missing+=("$package")
+done
+if (( ${#missing[@]} )); then
+    mkdir -m 0700 -- "$state"
+    trap restore_policy EXIT
+    trap 'exit 130' INT
+    trap 'exit 143' TERM
+    if [[ -e $policy || -L $policy ]]; then
+        cp -a -- "$policy" "$state/original"
+    else
+        touch "$state/absent"
+    fi
+    touch "$state/ready"
+    printf '#!/bin/sh\nexit 101\n' > "$state/guard"
+    chmod 0755 "$state/guard"
+    mv -Tf -- "$state/guard" "$policy"
+    apt=(apt-get -o DPkg::Lock::Timeout=60 -o Acquire::Retries=2
+         -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
+    timeout 300 "${apt[@]}" update --error-on=any
+    install=("${apt[@]}" install -y --no-install-recommends --no-upgrade --no-remove "${missing[@]}")
+    plan=$(timeout 300 "${install[@]}" --simulate)
+    if grep -Eq '^Remv |^Inst [^ ]+ \[' <<< "$plan"; then
+        echo 'Parser installation requires a package upgrade/removal; reconcile it explicitly.' >&2
+        exit 1
+    fi
+    "${install[@]}"
+fi
+python3 -c 'import yaml'

ci/scripts/init-node/provisioning/state.py

@@ -0,0 +1,128 @@
+"""Durable enrollment records and writes that refuse redirected paths."""
+from __future__ import annotations
+
+import contextlib
+import fcntl
+import json
+import os
+from pathlib import Path
+import stat
+import tempfile
+import time
+
+from . import ENROLLMENT_INTERFACE
+
+
+class ProvisionError(RuntimeError):
+    pass
+
+
+def safe_path(path):
+    path = Path(path).absolute()
+    for part in (*reversed(path.parents), path):
+        try:
+            info = part.lstat()
+        except FileNotFoundError:
+            continue
+        if stat.S_ISLNK(info.st_mode):
+            raise ProvisionError('managed path contains a symbolic link')
+    return path
+
+
+def atomic_write(path, content, mode=0o600, uid=None, gid=None):
+    path = safe_path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists() and (not path.is_file() or path.stat().st_nlink != 1):
+        raise ProvisionError('managed file is not an ordinary unlinked file')
+    data = content.encode() if isinstance(content, str) else content
+    fd, temporary = tempfile.mkstemp(prefix='.apex-', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'wb') as stream:
+            os.fchmod(stream.fileno(), mode)
+            if uid is not None:
+                os.fchown(stream.fileno(), uid, gid)
+            stream.write(data)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+        directory = os.open(path.parent, os.O_RDONLY)
+        try:
+            os.fsync(directory)
+        finally:
+            os.close(directory)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    path = safe_path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+    try:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise ProvisionError('another initialization is running; inspect status before retrying') from None
+        yield
+    finally:
+        os.close(fd)
+
+
+def check_heartbeat(path, now=None):
+    if path is None:
+        return
+    try:
+        age = (time.time() if now is None else now) - safe_path(path).stat().st_mtime
+    except FileNotFoundError:
+        raise ProvisionError('coordinator heartbeat is missing; stopping before the next stage') from None
+    if not 0 <= age < 90:
+        raise ProvisionError('coordinator heartbeat expired; stopping before the next stage')
+
+
+class Journal:
+    def __init__(self, path, identity, revision, bundle_digest):
+        self.path = safe_path(path)
+        if self.path.exists():
+            try:
+                self.data = json.loads(self.path.read_text())
+            except (ValueError, OSError):
+                raise ProvisionError('invalid enrollment journal; reconcile it before retrying') from None
+            if self.data.get('enrollment_interface') != ENROLLMENT_INTERFACE:
+                raise ProvisionError('recorded enrollment interface is missing or unsupported')
+            if self.data.get('identity') != identity:
+                raise ProvisionError('enrollment identity conflicts with the recorded target')
+            if self.data.get('bundle_digest') != bundle_digest:
+                raise ProvisionError('provisioner bundle changed; resume using the recorded bundle')
+        else:
+            self.data = dict(identity=identity, revision=revision, bundle_digest=bundle_digest,
+                             baseline_revision=revision, enrollment_interface=ENROLLMENT_INTERFACE,
+                             status='running', stages={})
+
+    def save(self):
+        atomic_write(self.path, json.dumps(self.data, sort_keys=True, indent=2) + '\n')
+
+    def stage(self, name, operation, heartbeat=None):
+        try:
+            check_heartbeat(heartbeat)
+        except ProvisionError:
+            self.data['status'] = 'stopped'
+            self.data['failed_stage'] = name
+            self.save()
+            raise
+        self.data['status'] = 'running'
+        self.data['stages'][name] = {'status': 'running'}
+        self.save()
+        try:
+            result = operation()
+        except Exception:
+            self.data['status'] = 'failed'
+            self.data['failed_stage'] = name
+            self.data['stages'][name] = {'status': 'failed'}
+            self.save()
+            raise
+        self.data['stages'][name] = {'status': 'done', 'result': result}
+        self.data.pop('failed_stage', None)
+        self.save()
+        return result

ci/scripts/init-node/provisioning/storage.py

@@ -0,0 +1,509 @@
+"""Conservative target-side tier and optional swap reconciliation.
+
+The caller holds the initialization lock and supplies a zero-argument atomic
+journal writer. Ambiguous device or pool ownership always requires remediation.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import tempfile
+
+from .contract import Enrollment
+
+
+SRV = Path('/srv')
+SWAPFILE = Path('/swapfile')
+FSTAB = Path('/etc/fstab')
+MEMINFO = Path('/proc/meminfo')
+SWAPS = Path('/proc/swaps')
+_OWNER = 'org.apex:enrollment'
+_GIB = 1024 ** 3
+
+
+class StorageError(RuntimeError):
+    """Storage could not be positively identified or safely reconciled."""
+
+
+def _require(condition, message):
+    if not condition:
+        raise StorageError(message)
+
+
+def _run(argv, *, full_vdev_paths=False):
+    environment = {**os.environ, 'LC_ALL': 'C'}
+    if full_vdev_paths:
+        environment.update(ZPOOL_VDEV_NAME_PATH='1', ZPOOL_VDEV_NAME_FOLLOW_LINKS='1')
+        environment.pop('ZPOOL_VDEV_NAME_GUID', None)
+    try:
+        result = subprocess.run(argv, check=False, capture_output=True, text=True,
+                                timeout=300, env=environment)
+    except (OSError, subprocess.TimeoutExpired):
+        raise StorageError('required storage tool is unavailable or timed out: ' + argv[0]) from None
+    _require(result.returncode == 0,
+             'storage inspection or operation failed: ' + ' '.join(argv[:2]))
+    return result.stdout
+
+
+def _json(argv, key):
+    try:
+        value = json.loads(_run(argv))[key]
+        _require(isinstance(value, list), 'storage inspection returned an invalid list')
+        return value
+    except (ValueError, KeyError, TypeError):
+        raise StorageError('storage inspection returned unsupported JSON') from None
+
+
+def _flatten(items):
+    for item in items:
+        yield item
+        yield from _flatten(item.get('children', []))
+
+
+def _mounts():
+    return list(_flatten(_json(['findmnt', '--json', '--output', 'TARGET,SOURCE,FSTYPE,MAJ:MIN'],
+                               'filesystems')))
+
+
+def _device_number(path):
+    try:
+        info = os.stat(path)
+    except OSError:
+        raise StorageError('selected device is missing or is not a block device') from None
+    _require(stat.S_ISBLK(info.st_mode), 'selected device is not a block device')
+    return f'{os.major(info.st_rdev)}:{os.minor(info.st_rdev)}'
+
+
+def _inventory():
+    roots = _json(['lsblk', '--json', '--bytes', '--paths', '--output',
+                   'NAME,MAJ:MIN,TYPE,PKNAME,MOUNTPOINTS,FSTYPE,WWN,SERIAL,SIZE,PARTUUID'], 'blockdevices')
+    nodes, edges = {}, {}
+
+    def visit(items, parent=None):
+        for item in items:
+            number = item['maj:min']
+            nodes[number] = item
+            edges.setdefault(number, set())
+            if parent:
+                edges[number].add(parent)
+                edges[parent].add(number)
+            visit(item.get('children', []), number)
+
+    try:
+        visit(roots)
+        by_name = {item['name']: number for number, item in nodes.items()}
+        for number, item in nodes.items():
+            parent = by_name.get(item.get('pkname'))
+            if parent:
+                edges[number].add(parent)
+                edges[parent].add(number)
+    except (KeyError, TypeError):
+        raise StorageError('block-device topology could not be established') from None
+    return nodes, edges
+
+
+def _connected(number, edges):
+    found, pending = set(), [number]
+    while pending:
+        item = pending.pop()
+        if item not in found:
+            found.add(item)
+            pending.extend(edges.get(item, ()))
+    return found
+
+
+def _pools():
+    result = {}
+    for line in _run(['zpool', 'list', '-H', '-o', 'name,guid,health']).splitlines():
+        fields = line.split('\t')
+        _require(len(fields) == 3 and fields[1].isdigit(), 'active pool identity is ambiguous')
+        result[fields[0]] = {'guid': fields[1], 'health': fields[2]}
+    return result
+
+
+def _members(text, name, imported=True):
+    """Accept only a plain single-device ONLINE pool, never infer topology."""
+    marker = re.search(r'^\s*config:\s*$', text, re.M)
+    _require(marker is not None, 'pool membership is unavailable')
+    lines = text[marker.end():].splitlines()
+    rows = []
+    for line in lines:
+        fields = line.split()
+        if not fields or fields[0] == 'NAME':
+            continue
+        if fields[0] == 'errors:':
+            _require(line.strip() == 'errors: No known data errors', 'pool reports data errors')
+            break
+        _require(len(fields) >= 2 and fields[1] == 'ONLINE',
+                 'pool is not a healthy single-device topology')
+        if imported:
+            _require(len(fields) == 5 and fields[2:] == ['0', '0', '0'],
+                     'pool reports device errors')
+        rows.append(fields[0])
+    _require(len(rows) == 2 and rows[0] == name,
+             'pool must contain exactly one identifiable device')
+    leaf = rows[1]
+    if not imported and re.fullmatch(r'[A-Za-z0-9_.-]+', leaf):
+        leaf = '/dev/' + leaf
+    _require(leaf.startswith('/dev/'), 'pool device path cannot be positively resolved')
+    return leaf
+
+
+def _pool_device(name):
+    return _members(_run(['zpool', 'status', '-P', name], full_vdev_paths=True), name)
+
+
+def _inspect_device(device, mounts, allowed_pool=None, allowed_path=None):
+    number = _device_number(device)
+    nodes, edges = _inventory()
+    _require(number in nodes, 'selected device is absent from the block topology')
+    node = nodes[number]
+    hardware = node.get('wwn') or node.get('serial')
+    partition = None
+    if node.get('type') == 'part':
+        parents = [item for item in nodes.values() if item['name'] == node.get('pkname')]
+        _require(len(parents) == 1 and parents[0]['type'] == 'disk' and node.get('partuuid'),
+                 'selected partition needs a stable partition UUID and identifiable parent disk')
+        hardware = parents[0].get('wwn') or parents[0].get('serial')
+        partition = node['partuuid']
+    _require(isinstance(hardware, str) and bool(hardware.strip()),
+             'selected device needs a stable WWN or hardware serial')
+    _require(node.get('type') in ('disk', 'part') and int(node.get('size') or 0) > 0,
+             'selected device type or capacity is unsupported')
+    related = _connected(number, edges)
+    protected = set()
+    for other, data in nodes.items():
+        for mount in data.get('mountpoints') or []:
+            if mount and mount != allowed_path:
+                protected.add(other)
+    for mount in mounts:
+        if mount['target'] != allowed_path:
+            protected.add(mount.get('maj:min'))
+    active = _pools()
+    for pool in active:
+        leaf = _device_number(_pool_device(pool))
+        if pool != allowed_pool or any(
+                m['source'].split('/')[0] == pool and m['target'] != allowed_path for m in mounts):
+            protected.add(leaf)
+    _require(not (related & protected),
+             'selected device intersects mounted, root, boot, swap, or active pool storage')
+    identity = {'hardware': hardware, 'size': int(node['size']), 'type': node['type']}
+    if partition:
+        identity['partition'] = partition
+    return {'number': number, 'identity': identity, 'related': related, 'node': node, 'pools': active}
+
+
+def _safe_path(path):
+    for parent in reversed((path, *path.parents)):
+        try:
+            info = parent.lstat()
+        except FileNotFoundError:
+            _require(parent == path, 'storage parent directory is missing')
+            continue
+        _require(stat.S_ISDIR(info.st_mode), 'storage path contains a symlink or non-directory')
+
+
+def _directory(path, mounts, pool=None):
+    _safe_path(path)
+    exact = [m for m in mounts if m['target'] == str(path)]
+    _require(not exact or (len(exact) == 1 and pool is not None and
+                          exact[0]['fstype'] == 'zfs' and exact[0]['source'] == pool),
+             'tier path is occupied by an unexpected mount')
+    _require(not any(m['target'].startswith(str(path) + '/') for m in mounts),
+             'tier path contains an unexpected nested mount')
+    created = not path.exists()
+    if created:
+        path.mkdir(mode=0o755)
+        path.chmod(0o755)
+    return created
+
+
+def _directory_identity(path):
+    _safe_path(path)
+    try:
+        info = path.lstat()
+    except FileNotFoundError:
+        raise StorageError('managed directory identity is missing') from None
+    return {'device': info.st_dev, 'inode': info.st_ino}
+
+
+def _verify_ownership_identity(record, path):
+    ownership = record.get('ownership')
+    if ownership:
+        _require(ownership['identity'] == _directory_identity(path),
+                 'managed directory identity changed before ownership handoff')
+    return ownership
+
+
+def _record_pending_ownership(record, path):
+    if not _verify_ownership_identity(record, path):
+        record['ownership'] = {'status': 'pending', 'identity': _directory_identity(path)}
+
+
+def _verify_pool(name, guid, device_info, path, owner, require_owner=True):
+    pools = _pools()
+    _require(name in pools and pools[name] == {'guid': guid, 'health': 'ONLINE'},
+             'pool name, GUID, or health does not match enrollment')
+    _require(_member_matches(_pool_device(name), device_info),
+             'pool device membership does not match the selected device')
+    rows = [line.split('\t') for line in _run(
+        ['zfs', 'list', '-H', '-r', '-o', 'name,mountpoint,type,mounted', name]).splitlines()]
+    _require(len(rows) == 1 and len(rows[0]) == 4 and rows[0][:3] == [name, str(path), 'filesystem'],
+             'pool dataset or mountpoint requires explicit reconciliation')
+    if require_owner:
+        actual = _run(['zfs', 'get', '-H', '-o', 'value', _OWNER, name]).strip()
+        _require(actual == owner, 'pool lacks matching enrollment ownership')
+    return rows[0][3] == 'yes'
+
+
+def _member_matches(member, info):
+    number = _device_number(member)
+    return number == info['number'] or (info['node']['type'] == 'disk' and number in info['related'])
+
+
+def _import_candidate(device, guid):
+    text = _run(['zpool', 'import', '-d', device], full_vdev_paths=True)
+    names = re.findall(r'^\s*pool:\s*(\S+)\s*$', text, re.M)
+    guids = re.findall(r'^\s*id:\s*(\d+)\s*$', text, re.M)
+    states = re.findall(r'^\s*state:\s*(\S+)\s*$', text, re.M)
+    _require(len(names) == 1 and guids == [guid] and states == ['ONLINE'],
+             'import discovery does not match the declared healthy pool GUID')
+    actions = re.findall(r'^\s*action:\s*(.+)$', text, re.M)
+    _require(not re.search(r'^\s*status:', text, re.M) and
+             all(action == 'The pool can be imported using its name or numeric identifier.' for action in actions),
+             'pool import reports ownership or recovery conditions; reconcile manually')
+    return names[0], _members(text, names[0], imported=False)
+
+
+def _block_tier(tier, path, journal, owner, save):
+    record = journal.get(str(tier.number))
+    created = False
+    pool = record.get('pool') if record else None
+    info = _inspect_device(tier.device, _mounts(), pool, str(path) if pool else None)
+    if record:
+        _require(record['device'] == info['identity'], 'selected device identity changed since enrollment')
+        _require(record['status'] in ('intent', 'complete'), 'tier journal status is unsupported')
+    if pool and pool in info['pools']:
+        guid = record.get('guid') or info['pools'][pool]['guid']
+        _verify_pool(pool, guid, info, path, owner)
+    else:
+        _require(not record or record['status'] == 'intent',
+                 'previously enrolled pool is not active; explicit reconciliation is required')
+        if tier.mode == 'create':
+            pool = f'tier-{tier.number}'
+            _require(pool not in info['pools'], 'existing pool name does not prove enrollment ownership')
+            _require(info['node']['type'] == 'disk', 'pool creation requires an unused whole disk')
+            _require(len(info['related']) == 1, 'selected disk has partitions or dependent devices')
+            signatures = _json(['wipefs', '--json', '--output', 'TYPE,UUID,LABEL', tier.device], 'signatures')
+            _require(not signatures and not info['node'].get('fstype'), 'selected device contains existing signatures')
+        else:
+            pool, member = _import_candidate(tier.device, tier.pool_guid)
+            _require(_member_matches(member, info), 'import membership differs from the selected device')
+            _require(pool not in info['pools'], 'pool is already active without matching enrollment ownership')
+        created = _directory(path, _mounts())
+        _require(not path.exists() or not any(path.iterdir()), 'pool mountpoint contains existing data')
+        record = {'status': 'intent', 'device': info['identity'], 'pool': pool,
+                  'ownership_intent': tier.mode == 'create'}
+        journal[str(tier.number)] = record
+        save()
+        fresh = _inspect_device(tier.device, _mounts())
+        _require(fresh['identity'] == info['identity'] and fresh['number'] == info['number'],
+                 'selected device identity changed before pool operation')
+        _directory(path, _mounts())
+        _require(not any(path.iterdir()), 'pool mountpoint acquired existing data before operation')
+        if tier.mode == 'create':
+            _require(not _json(['wipefs', '--json', '--output', 'TYPE,UUID,LABEL', tier.device], 'signatures'),
+                     'selected device acquired signatures before creation')
+            _run(['zpool', 'create', '-o', 'ashift=12', '-O', f'mountpoint={path}',
+                  '-O', f'{_OWNER}={owner}', pool, tier.device])
+        else:
+            _require(_import_candidate(tier.device, tier.pool_guid) == (pool, member),
+                     'pool identity changed before import')
+            _run(['zpool', 'import', '-N', '-d', tier.device, tier.pool_guid])
+        pools = _pools()
+        _require(pool in pools, 'pool operation did not establish an active pool')
+        guid = pools[pool]['guid']
+        _require(tier.mode != 'import' or guid == tier.pool_guid, 'imported pool GUID differs from intent')
+        refreshed = _inspect_device(tier.device, _mounts(), pool, str(path))
+        _require(refreshed['identity'] == info['identity'], 'selected device changed during pool operation')
+        info = refreshed
+        _verify_pool(pool, guid, info, path, owner, require_owner=tier.mode == 'create')
+        if tier.mode == 'import':
+            previous_owner = _run(['zfs', 'get', '-H', '-o', 'value', _OWNER, pool]).strip()
+            _require(previous_owner in ('-', owner), 'imported pool belongs to a different enrollment')
+            _run(['zfs', 'set', f'{_OWNER}={owner}', pool])
+    created = _directory(path, _mounts(), pool) or created
+    if not _verify_pool(pool, guid, info, path, owner):
+        _require(not any(path.iterdir()), 'refusing to hide existing files beneath a pool mount')
+        _run(['zfs', 'mount', pool])
+    _require(any(m['target'] == str(path) and m['source'] == pool and m['fstype'] == 'zfs'
+                 for m in _mounts()), 'pool is not mounted at its enrolled path')
+    record.update(status='complete', guid=guid)
+    if tier.mode == 'create' and record.get('ownership_intent'):
+        _record_pending_ownership(record, path)
+    save()
+    return {'path': str(path), 'mode': tier.mode, 'created': created and tier.mode == 'create',
+            'pool': pool, 'guid': guid}
+
+
+def _persist_swap():
+    info = FSTAB.lstat()
+    _require(stat.S_ISREG(info.st_mode) and info.st_uid == 0, 'fstab must be a root-owned regular file')
+    original = FSTAB.read_text()
+    lines = [line.split() for line in original.splitlines() if line.strip() and not line.lstrip().startswith('#')]
+    matches = [line for line in lines if line[0] == str(SWAPFILE)]
+    if matches:
+        _require(len(matches) == 1 and len(matches[0]) >= 4 and matches[0][2] == 'swap'
+                 and 'noauto' not in matches[0][3].split(','),
+                 'existing swap fstab entry is incompatible')
+        return
+    fd, temporary = tempfile.mkstemp(prefix='.apex-swap-', dir=FSTAB.parent)
+    try:
+        with os.fdopen(fd, 'w') as output:
+            os.fchmod(output.fileno(), stat.S_IMODE(info.st_mode))
+            os.fchown(output.fileno(), info.st_uid, info.st_gid)
+            output.write(original + ('' if original.endswith('\n') else '\n') +
+                         f'{SWAPFILE} none swap sw 0 0\n')
+            output.flush()
+            os.fsync(output.fileno())
+        current = FSTAB.lstat()
+        _require(all(getattr(current, key) == getattr(info, key) for key in
+                     ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')),
+                 'fstab changed during swap setup')
+        os.replace(temporary, FSTAB)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def _active_swaps():
+    lines = SWAPS.read_bytes().splitlines()
+    _require(bool(lines) and lines[0].split() == [b'Filename', b'Type', b'Size', b'Used', b'Priority'],
+             'kernel swap inventory header is invalid')
+    paths = []
+    for line in lines[1:]:
+        fields = line.split()
+        _require(len(fields) == 5 and fields[0].startswith(b'/') and fields[1] in (b'file', b'partition')
+                 and fields[2].isdigit() and fields[3].isdigit()
+                 and re.fullmatch(rb'-?[0-9]+', fields[4]) is not None,
+                 'kernel swap inventory entry is invalid')
+        _require(re.search(rb'\\(?![0-3][0-7]{2})', fields[0]) is None,
+                 'kernel swap inventory path escaping is invalid')
+        decoded = re.sub(rb'\\([0-3][0-7]{2})', lambda match: bytes([int(match.group(1), 8)]), fields[0])
+        paths.append(os.fsdecode(decoded))
+    return paths
+
+
+def _swap(journal, save):
+    try:
+        active = _active_swaps()
+        if active:
+            return {'status': 'skipped', 'reason': 'existing active swap preserved'}
+        if os.path.lexists(SWAPFILE):
+            return {'status': 'skipped', 'reason': 'existing swapfile preserved; no overwrite or resize'}
+        mounts = _mounts()
+        covering = [m for m in mounts if str(SWAPFILE).startswith(m['target'].rstrip('/') + '/')]
+        _require(bool(covering), 'swap filesystem could not be identified')
+        filesystem = max(covering, key=lambda m: len(m['target']))['fstype']
+        _require(filesystem in ('ext4', 'xfs'), 'optional swap requires a supported ext4 or XFS filesystem')
+        match = re.search(r'^MemTotal:\s+(\d+)\s+kB$', MEMINFO.read_text(), re.M)
+        _require(match is not None, 'RAM size could not be established')
+        size = (4 if int(match.group(1)) * 1024 > 4 * _GIB else 2) * _GIB
+        space = os.statvfs(SWAPFILE.parent)
+        _require(space.f_bavail * space.f_frsize >= size + _GIB,
+                 'insufficient free space for optional swap and reserve')
+        _require(hasattr(os, 'posix_fallocate'), 'safe swap allocation is unavailable')
+        journal['swap'] = {'status': 'intent', 'size': size}
+        save()
+        fd = os.open(SWAPFILE, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+        try:
+            os.posix_fallocate(fd, 0, size)
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+        _run(['mkswap', str(SWAPFILE)])
+        _run(['swapon', str(SWAPFILE)])
+        active = _active_swaps()
+        _require(str(SWAPFILE) in active,
+                 'optional swap activation did not become observable')
+        _persist_swap()
+        return {'status': 'complete', 'size': size}
+    except (StorageError, OSError) as error:
+        return {'status': 'skipped', 'reason': str(error) if isinstance(error, StorageError)
+                else 'optional swap allocation or persistence failed; existing resources preserved'}
+
+
+def _enrollment_identity(spec):
+    return hashlib.sha256(json.dumps(spec.identity(), sort_keys=True,
+                                     separators=(',', ':')).encode()).hexdigest()
+
+
+def acknowledge_storage_ownership(spec: Enrollment, state: dict, paths, save) -> None:
+    """Persist the caller's successful, nonrecursive ownership handoff for paths."""
+    journal = state.get('storage', {})
+    _require(journal.get('identity') == _enrollment_identity(spec),
+             'storage enrollment identity conflicts with ownership acknowledgement')
+    allowed = {str(SRV / f'tier-{tier.number}.{spec.repository.name}'): tier for tier in spec.tiers}
+    acknowledged = []
+    for path in paths:
+        _require(path in allowed, 'ownership acknowledgement path is outside enrollment')
+        tier = allowed[path]
+        record = journal.get('tiers', {}).get(str(tier.number), {})
+        ownership = _verify_ownership_identity(record, Path(path))
+        _require(tier.mode != 'import' and ownership is not None,
+                 'ownership acknowledgement requires a directory created by enrollment')
+        _require(ownership['status'] in ('pending', 'applied'), 'ownership journal status is unsupported')
+        acknowledged.append(ownership)
+    for ownership in acknowledged:
+        ownership['status'] = 'applied'
+    save()
+
+
+def reconcile_storage(spec: Enrollment, state: dict, save) -> dict:
+    """Return ownership_pending paths until the caller acknowledges their chown."""
+    identity = _enrollment_identity(spec)
+    journal = state.setdefault('storage', {'identity': identity, 'tiers': {}})
+    _require(journal.get('identity') == identity, 'storage enrollment identity conflicts with journal')
+    _safe_path(SRV)
+    _require(SRV.exists(), 'storage parent directory is missing')
+    selected = set()
+    for tier in spec.tiers:
+        if tier.device:
+            number = _device_number(tier.device)
+            _require(number not in selected, 'multiple tiers select aliases of the same device')
+            selected.add(number)
+    result = {'tiers': [], 'ownership_pending': []}
+    for tier in spec.tiers:
+        path = SRV / f'tier-{tier.number}.{spec.repository.name}'
+        if tier.mode == 'directory':
+            record = journal['tiers'].get(str(tier.number), {})
+            _verify_ownership_identity(record, path)
+            _safe_path(path)
+            _require(not (record.get('ownership_intent') and not record.get('ownership') and path.exists()),
+                     'directory creation was interrupted before identity was recorded; reconcile manually')
+            if not path.exists():
+                record = {'status': 'intent', 'path': str(path), 'ownership_intent': True}
+                journal['tiers'][str(tier.number)] = record
+                save()
+            created = _directory(path, _mounts())
+            if created:
+                _record_pending_ownership(record, path)
+            entry = {'path': str(path), 'mode': tier.mode, 'created': created}
+            record.update(status='complete', path=str(path))
+            journal['tiers'][str(tier.number)] = record
+            save()
+        else:
+            entry = _block_tier(tier, path, journal['tiers'], identity, save)
+        ownership = journal['tiers'][str(tier.number)].get('ownership')
+        if ownership and ownership['status'] == 'pending':
+            result['ownership_pending'].append(str(path))
+        result['tiers'].append(entry)
+    result['swap'] = _swap(journal, save)
+    journal['swap'] = result['swap']
+    save()
+    return result

ci/scripts/init-node/provisioning/target.py

@@ -0,0 +1,170 @@
+"""Shared root runner for remote and cloud-init enrollment."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import platform
+import pwd
+import subprocess
+import sys
+import tempfile
+
+from .state import Journal, ProvisionError, atomic_write, exclusive_lock, safe_path
+from .userdata import decode_user_data
+
+STATE = Path('/var/lib/apex-init')
+
+
+def run(args, **kwargs):
+    result = subprocess.run(args, text=True, capture_output=True, timeout=300, **kwargs)
+    if result.returncode:
+        raise ProvisionError('required target operation failed: ' + args[0])
+    return result.stdout.strip()
+
+
+def preflight():
+    if os.geteuid() != 0:
+        raise ProvisionError('target enrollment requires root privileges')
+    release = {}
+    for line in Path('/etc/os-release').read_text().splitlines():
+        key, separator, value = line.partition('=')
+        if separator:
+            release[key] = value.strip('"')
+    if release.get('ID') != 'debian' or release.get('VERSION_ID') != '13' or platform.machine() != 'x86_64':
+        raise ProvisionError('initialization supports Debian 13 amd64 only')
+
+
+def validate_keys(spec):
+    for key in spec.authorized_keys:
+        with tempfile.NamedTemporaryFile(mode='w') as stream:
+            stream.write(key + '\n')
+            stream.flush()
+            try:
+                run(['ssh-keygen', '-l', '-f', stream.name])
+            except ProvisionError:
+                raise ProvisionError('administrator public key failed cryptographic validation') from None
+
+
+def administrator(spec):
+    try:
+        account = pwd.getpwnam('adam')
+    except KeyError:
+        run(['useradd', '--create-home', '--shell', '/bin/bash', '--groups', 'sudo,docker', 'adam'])
+        account = pwd.getpwnam('adam')
+    if account.pw_dir != '/home/adam' or account.pw_uid == 0:
+        raise ProvisionError('existing administrator has incompatible home or UID')
+    safe_path(account.pw_dir)
+    if account.pw_shell != '/bin/bash':
+        run(['usermod', '--shell', '/bin/bash', 'adam'])
+    run(['usermod', '--append', '--groups', 'sudo,docker', 'adam'])
+    directory = safe_path(Path(account.pw_dir) / '.ssh')
+    directory.mkdir(mode=0o700, exist_ok=True)
+    os.chown(directory, account.pw_uid, account.pw_gid)
+    os.chmod(directory, 0o700)
+    authorized = safe_path(directory / 'authorized_keys')
+    existing = authorized.read_text().splitlines() if authorized.exists() else []
+    keys = existing + [key for key in spec.authorized_keys if key not in existing]
+    atomic_write(authorized, '\n'.join(keys) + '\n', uid=account.pw_uid, gid=account.pw_gid)
+    policy = 'adam ALL=(ALL:ALL) NOPASSWD: ALL\n'
+    with tempfile.NamedTemporaryFile(mode='w') as stream:
+        stream.write(policy)
+        stream.flush()
+        run(['visudo', '-cf', stream.name])
+    atomic_write('/etc/sudoers.d/99-apex-adam', policy, 0o440)
+    run(['visudo', '-c'])
+    run(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true'])
+    for item in spec.files:
+        uid, gid = (0, 0) if item.owner == 'root:root' else (account.pw_uid, account.pw_gid)
+        atomic_write(item.path, item.content, item.mode, uid=uid, gid=gid)
+    if run(['hostname']) != spec.hostname:
+        run(['hostnamectl', 'set-hostname', spec.hostname])
+    groups = run(['id', '-nG', 'adam']).split()
+    if not {'sudo', 'docker'} <= set(groups):
+        raise ProvisionError('administrator groups did not converge')
+    return {'administrator': 'adam', 'hostname': spec.hostname, 'sudo': 'verified'}
+
+
+def enroll(spec, revision, bundle_digest, heartbeat=None):
+    from .baseline import apply_baseline
+    from .storage import acknowledge_storage_ownership, reconcile_storage
+    from .repository import reconcile_repository
+
+    preflight()
+    validate_keys(spec)
+    safe_path(STATE).mkdir(mode=0o700, exist_ok=True)
+    if STATE.stat().st_uid != 0:
+        raise ProvisionError('enrollment state directory must be root-owned')
+    os.chmod(STATE, 0o700)
+    with exclusive_lock(STATE / 'init.lock'):
+        journal = Journal(STATE / 'enrollment.json', spec.identity(), revision, bundle_digest)
+        journal.save()
+        journal.stage('baseline', apply_baseline, heartbeat)
+        journal.stage('administrator', lambda: administrator(spec), heartbeat)
+        result = journal.stage('storage', lambda: reconcile_storage(spec, journal.data, journal.save), heartbeat)
+        account = pwd.getpwnam('adam')
+        for path in result['ownership_pending']:
+            os.chown(safe_path(path), account.pw_uid, account.pw_gid)
+            acknowledge_storage_ownership(spec, journal.data, [path], journal.save)
+        journal.stage('repository', lambda: reconcile_repository(spec, journal.data, journal.save), heartbeat)
+        journal.data['status'] = 'enrolled'
+        journal.data['external_access'] = 'pending'
+        journal.save()
+        return journal.data
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('operation', choices=('validate', 'enroll', 'status', 'complete'))
+    parser.add_argument('--user-data')
+    parser.add_argument('--revision', default='unknown')
+    parser.add_argument('--bundle-digest', default='unknown')
+    parser.add_argument('--heartbeat')
+    args = parser.parse_args(argv)
+    try:
+        preflight()
+        if args.operation == 'status':
+            path = safe_path(STATE / 'enrollment.json')
+            print(path.read_text() if path.exists() else json.dumps({'status': 'not-started'}))
+            return 0
+        if args.operation == 'complete':
+            with exclusive_lock(STATE / 'init.lock'):
+                access = json.loads(safe_path(STATE / 'access/state.json').read_text())
+                path = safe_path(STATE / 'enrollment.json')
+                record = json.loads(path.read_text())
+                if access.get('status') != 'committed' or record.get('status') not in ('enrolled', 'complete'):
+                    raise ProvisionError('enrollment and access must be verified before completion')
+                record['external_access'] = 'verified'
+                record['status'] = 'complete'
+                atomic_write(path, json.dumps(record, sort_keys=True) + '\n')
+                print(json.dumps(record, sort_keys=True))
+            return 0
+        if not args.user_data:
+            raise ProvisionError('--user-data is required')
+        path = safe_path(args.user_data)
+        if path.stat().st_uid != 0 or path.stat().st_mode & 0o077:
+            raise ProvisionError('user-data must be root-owned and private')
+        if not path.is_file() or path.stat().st_size > 1024 * 1024:
+            raise ProvisionError('user-data must be a regular file no larger than 1 MiB')
+        spec = decode_user_data(path.read_bytes())
+        validate_keys(spec)
+        if args.operation == 'validate':
+            print(json.dumps({'status': 'valid', 'hostname': spec.hostname}))
+        else:
+            print(json.dumps(enroll(spec, args.revision, args.bundle_digest, args.heartbeat), sort_keys=True))
+        return 0
+    except Exception as error:
+        from .contract import ContractError
+        from .baseline import BaselineError
+        from .storage import StorageError
+        if isinstance(error, (ProvisionError, ContractError, BaselineError, StorageError)):
+            print(str(error), file=sys.stderr)
+        else:
+            # Unexpected dependency failures must not render raw credential input.
+            print('initialization failed (' + type(error).__name__ + '); inspect stage status', file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

ci/scripts/init-node/provisioning/transport.py

@@ -0,0 +1,385 @@
+"""Workstation SSH transport. User-data stays opaque until target validation."""
+from __future__ import annotations
+
+import hashlib
+import io
+import ipaddress
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import time
+import uuid
+
+
+class TransportError(RuntimeError):
+    pass
+
+
+class HandoverUncertain(TransportError):
+    pass
+
+
+# This runs before Python is guaranteed to exist on the Debian target. All
+# ancestors must be trusted; a root-owned sticky temporary directory is safe
+# only because root owns each managed child and exclusive mktemp creates it.
+REMOTE_PATH_GUARD = r'''
+set -eu
+umask 077
+apex_guard_dir() {
+    apex_checked=$1
+    while :; do
+        test ! -L "$apex_checked" && test -d "$apex_checked" || exit 91
+        test "$(stat -c %u -- "$apex_checked")" = 0 || exit 91
+        apex_mode=$(stat -c %a -- "$apex_checked")
+        test "$((0$apex_mode & 0022))" = 0 || test "$((0$apex_mode & 01000))" != 0 || exit 91
+        test "$apex_checked" != / || break
+        apex_checked=${apex_checked%/*}
+        test -n "$apex_checked" || apex_checked=/
+    done
+}
+apex_ensure_dir() {
+    apex_new_dir=$1
+    if test -e "$apex_new_dir" || test -L "$apex_new_dir"; then
+        apex_guard_dir "$apex_new_dir"
+        apex_mode=$(stat -c %a -- "$apex_new_dir")
+        test "$((0$apex_mode & 0022))" = 0 || exit 91
+    else
+        apex_guard_dir "${apex_new_dir%/*}"
+        mkdir -m "$2" -- "$apex_new_dir"
+        apex_guard_dir "$apex_new_dir"
+    fi
+}
+apex_guard_file() {
+    apex_file=$1
+    apex_guard_dir "${apex_file%/*}"
+    apex_mode=$(stat -c %a -- "${apex_file%/*}")
+    test "$((0$apex_mode & 0022))" = 0 || exit 91
+    test ! -L "$apex_file" || exit 91
+    if test -e "$apex_file"; then
+        test -f "$apex_file" || exit 91
+        test "$(stat -c %u -- "$apex_file")" = 0 || exit 91
+        test "$(stat -c %h -- "$apex_file")" = 1 || exit 91
+        apex_mode=$(stat -c %a -- "$apex_file")
+        test "$((0$apex_mode & 0022))" = 0 || exit 91
+    fi
+}
+apex_atomic_input() {
+    apex_destination=$1
+    apex_guard_file "$apex_destination"
+    apex_temporary=$(mktemp "${apex_destination%/*}/.apex-write.XXXXXXXXXX")
+    trap 'rm -f -- "$apex_temporary"' EXIT HUP INT TERM
+    cat > "$apex_temporary"
+    chmod 0600 "$apex_temporary"
+    apex_guard_file "$apex_destination"
+    mv -T -- "$apex_temporary" "$apex_destination"
+    trap - EXIT HUP INT TERM
+}
+'''
+
+
+def remote_atomic_input(path):
+    return REMOTE_PATH_GUARD + '\napex_atomic_input ' + shlex.quote(path)
+
+
+def remote_bundle_install(stage, installed, digest):
+    return REMOTE_PATH_GUARD + '\n' + (
+        'apex_guard_dir ' + shlex.quote(stage) + '; apex_guard_file ' + shlex.quote(stage + '/bundle.tar') + '; '
+        'test "$(sha256sum ' + shlex.quote(stage + '/bundle.tar') + ' | cut -d " " -f 1)" = ' + digest + '; '
+        'apex_ensure_dir /usr/local/lib/apex-provisioner 0755; '
+        'if test -e ' + installed + ' || test -L ' + installed + '; then '
+        'apex_guard_dir ' + installed + '; '
+        'apex_bad=$(find ' + installed + ' -xdev \\( -type l -o ! -user root -o -perm /022 -o \\( ! -type d -a ! -type f \\) \\) -print -quit); test -z "$apex_bad"; '
+        'else apex_install_tmp=$(mktemp -d /usr/local/lib/apex-provisioner/.install.XXXXXXXXXX); '
+        'trap \'rm -rf -- "$apex_install_tmp"\' EXIT HUP INT TERM; '
+        'tar --no-same-owner --same-permissions -xf ' + stage + '/bundle.tar -C "$apex_install_tmp"; '
+        'mv -T -- "$apex_install_tmp" ' + installed + '; trap - EXIT HUP INT TERM; fi; '
+        'tar --compare -f ' + stage + '/bundle.tar -C ' + installed)
+
+
+def local_file(value, name):
+    if not value or any(ord(c) < 32 for c in str(value)):
+        raise TransportError(f'{name} must name a readable regular file')
+    try:
+        path = Path(value).expanduser().resolve(strict=True)
+    except (OSError, ValueError):
+        raise TransportError(f'{name} must name a readable regular file') from None
+    if not path.is_file() or not os.access(path, os.R_OK):
+        raise TransportError(f'{name} must name a readable regular file')
+    return path
+
+
+def destination(value, port=None):
+    match = re.fullmatch(r'([A-Za-z_][A-Za-z0-9_.-]{0,31})@([A-Za-z0-9][A-Za-z0-9_.:-]*|\[[0-9a-fA-F:]+\])', value or '')
+    if not match or '..' in match[2]:
+        raise TransportError('Target must be USER@HOST without SSH options; use root or an existing administrator with sudo -n')
+    hostname = match[2].strip('[]')
+    if ':' in hostname:
+        try:
+            ipaddress.IPv6Address(hostname)
+        except ValueError:
+            raise TransportError('IPv6 targets must contain a valid address; pass ports separately') from None
+    if port is not None and (not re.fullmatch(r'[0-9]{1,5}', str(port)) or not 1 <= int(port) <= 65535):
+        raise TransportError('Port must be an integer from 1 through 65535')
+    return match[1], hostname, int(port) if port is not None else None
+
+
+def build_bundle(commons):
+    root = Path(commons)
+    files = sorted((root / 'engine').rglob('*.py'))
+    seed = root / 'engine/provisioning/seed.sh'
+    if not seed.is_file():
+        raise TransportError('Provisioning seed is missing')
+    files.append(seed)
+    output = io.BytesIO()
+    with tarfile.open(fileobj=output, mode='w') as archive:
+        for path in files:
+            if path.is_symlink() or '__pycache__' in path.parts:
+                raise TransportError('Provisioning bundle contains an unsafe source path')
+            data = path.read_bytes()
+            info = tarfile.TarInfo(path.relative_to(root).as_posix())
+            info.size, info.mode, info.mtime = len(data), (0o755 if path == seed else 0o644), 0
+            info.uid = info.gid = 0
+            archive.addfile(info, io.BytesIO(data))
+    data = output.getvalue()
+    revision = 'unversioned-modified'
+    if shutil.which('git'):
+        result = subprocess.run(['git', '-C', str(root), 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=30)
+        if result.returncode == 0 and re.fullmatch(r'[a-f0-9]{40,64}\n?', result.stdout):
+            revision = result.stdout.strip()
+            dirty = subprocess.run(['git', '-C', str(root), 'status', '--porcelain', '--', 'engine'], capture_output=True, text=True, timeout=30)
+            if dirty.returncode or dirty.stdout.strip():
+                revision += '-modified'
+    return data, hashlib.sha256(data).hexdigest(), revision
+
+
+class SSH:
+    def __init__(self, target, port=None, identity_file=None, runner=subprocess.run):
+        self.user, self.host, self.port = destination(target, port)
+        self.identity = local_file(identity_file, 'Identity file') if identity_file else None
+        self.runner = runner
+        self.resolved_host = None
+        self.resolved_port = None
+        self.trust_file = None
+
+    def arguments(self, fresh=False):
+        args = ['ssh', '-T', '-o', 'StrictHostKeyChecking=yes', '-o', 'ConnectTimeout=15',
+                '-o', 'ServerAliveInterval=10', '-o', 'ServerAliveCountMax=3']
+        if self.identity:
+            args += ['-i', str(self.identity)]
+        args += ['-l', 'adam' if fresh else self.user]
+        if fresh:
+            if not self.trust_file or not self.resolved_host:
+                raise TransportError('Fresh access requires the verified bootstrap host identity')
+            args += ['-p', '2222', '-o', 'ControlMaster=no', '-o', 'ControlPath=none',
+                     '-o', 'ControlPersist=no', '-o', 'BatchMode=yes',
+                     '-o', 'PreferredAuthentications=publickey', '-o', 'PasswordAuthentication=no',
+                     '-o', 'KbdInteractiveAuthentication=no', '-o', 'Hostname=' + self.resolved_host,
+                     '-o', 'HostKeyAlias=apex-init-verified-target',
+                     '-o', 'UserKnownHostsFile=' + str(self.trust_file),
+                     '-o', 'GlobalKnownHostsFile=/dev/null']
+        elif self.port:
+            args += ['-p', str(self.port)]
+        return args
+
+    def resolve(self):
+        result = self.runner(self.arguments() + ['-G', '--', self.host], capture_output=True, text=True, timeout=30)
+        if result.returncode:
+            raise TransportError('Unable to resolve SSH connection configuration')
+        values = dict(line.split(' ', 1) for line in result.stdout.splitlines() if ' ' in line)
+        hostname = values.get('hostname', '')
+        port = values.get('port', '')
+        destination(self.user + '@' + hostname, port)
+        self.resolved_host, self.resolved_port = hostname, int(port)
+
+    def run(self, script, data=None, fresh=False, privileged=True, check=True, timeout=300):
+        command = ['sh', '-c', script]
+        if privileged and (fresh or self.user != 'root'):
+            command = ['sudo', '-n', '--', *command]
+        result = self.runner(self.arguments(fresh) + ['--', self.host, shlex.join(command)],
+                             input=data, capture_output=True, timeout=timeout)
+        if check and result.returncode:
+            # SSH and child diagnostics can contain user-data or credential-bearing URLs.
+            raise TransportError('Remote operation failed; inspect the root-owned target status for a redacted diagnosis')
+        return result
+
+    def fresh_login(self):
+        self.run('test "$(id -u)" = 0', fresh=True)
+
+
+def protected_handover(ssh, invoke, token, port, client_address):
+    result = invoke('begin', token, port, client_address, False)
+    try:
+        ssh.fresh_login()
+        if result['status'] == 'committed':
+            return
+        if result['status'] != 'existing':
+            invoke('finalize', token, None, None, True)
+            ssh.fresh_login()
+        try:
+            invoke('commit', token, None, None, True)
+        except Exception:
+            try:
+                state = invoke('status', token, None, None, True)
+                if state.get('token') != token or state.get('status') not in ('committed', 'existing', 'transition', 'final', 'rolled_back', 'recovery_pending'):
+                    raise TransportError('Unexpected access transition state')
+            except Exception:
+                raise HandoverUncertain('Access commit outcome is unknown; verify the administrator and bootstrap endpoints before resuming') from None
+            if state['status'] != 'committed':
+                raise
+    except HandoverUncertain:
+        raise
+    except Exception:
+        try:
+            invoke('rollback', token, None, None, True)
+        except Exception:
+            try:
+                invoke('rollback', token, None, None, False)
+            except Exception:
+                pass  # Persistent timer is authoritative if both connections are gone.
+        raise
+
+
+def initialize(commons, target, user_data, port=None, identity_file=None, progress=print):
+    if not shutil.which('ssh'):
+        raise TransportError('OpenSSH ssh is required on the workstation')
+    source = local_file(user_data, 'User-data')
+    if source.stat().st_size > 1024 * 1024:
+        raise TransportError('User-data must be no larger than 1 MiB')
+    ssh = SSH(target, port, identity_file)
+    ssh.resolve()
+    bundle, digest, revision = build_bundle(commons)
+    run_id = uuid.uuid4().hex
+    installed = '/usr/local/lib/apex-provisioner/' + digest
+    stage = None
+    launched = False
+    finished = False
+    access_committed = False
+    access_uncertain = False
+    cleanup_policy = '/etc/tmpfiles.d/apex-init-' + run_id + '.conf'
+    with tempfile.TemporaryDirectory(prefix='apex-init-trust-') as local:
+        try:
+            preflight = ssh.run('set -eu; test "$(id -u)" = 0; . /etc/os-release; '
+                                'test "$ID" = debian; test "$VERSION_ID" = 13; '
+                                'test "$(dpkg --print-architecture)" = amd64; '
+                                'command -v systemd-run >/dev/null; '
+                                'printf "%s\\n" "${SSH_CONNECTION:-}"; '
+                                'cat /etc/ssh/ssh_host_*_key.pub')
+            lines = preflight.stdout.decode().splitlines()
+            connection = lines[0].split() if lines else []
+            if len(connection) != 4:
+                # sudo commonly clears SSH_CONNECTION; read it before elevation instead.
+                raw = ssh.run('printf "%s\\n" "$SSH_CONNECTION"', privileged=False).stdout.decode().split()
+                connection = raw
+            if len(connection) != 4:
+                raise TransportError('Unable to inspect the established SSH connection')
+            keys = [line for line in lines[1:] if re.fullmatch(r'(ssh-[\w-]+|ecdsa-[\w-]+) [A-Za-z0-9+/=]+(?: .*?)?', line)]
+            if not keys:
+                raise TransportError('Unable to obtain public host keys through the verified connection')
+            trust = Path(local) / 'known_hosts'
+            trust.write_text(''.join('apex-init-verified-target,[apex-init-verified-target]:2222 ' + ' '.join(key.split()[:2]) + '\n' for key in keys))
+            trust.chmod(0o600)
+            ssh.trust_file = trust
+            existing = ssh.run(REMOTE_PATH_GUARD + '\napex_ensure_dir /var/lib/apex-init 0700; '
+                               'apex_guard_file /var/lib/apex-init/remote.json; '
+                               'if test -f /var/lib/apex-init/remote.json; then cat /var/lib/apex-init/remote.json; fi').stdout
+            if existing:
+                prior = json.loads(existing)
+                previous_unit = prior.get('unit', '')
+                if not re.fullmatch(r'apex-init-[a-f0-9]{32}', previous_unit):
+                    raise TransportError('Existing remote coordinator state requires manual inspection')
+                active = ssh.run(shlex.join(['systemctl', 'is-active', '--quiet', previous_unit]), check=False)
+                if active.returncode == 0:
+                    raise TransportError('An enrollment unit is still running: ' + previous_unit + '; wait for it to settle before resuming')
+            stage = ssh.run(REMOTE_PATH_GUARD + '\napex_guard_dir /var/tmp; mktemp -d /var/tmp/apex-init.XXXXXXXXXX').stdout.decode().strip()
+            if not re.fullmatch(r'/var/tmp/apex-init\.[A-Za-z0-9]{10}', stage):
+                raise TransportError('Target returned an invalid staging path')
+            # Hard reboot can bypass the shell trap; boot-time tmpfiles removes opaque inputs.
+            cleanup = ('r ' + stage + '/user-data - - - -\n' +
+                       'r ' + stage + '/heartbeat - - - -\n').encode()
+            ssh.run(REMOTE_PATH_GUARD + '\napex_ensure_dir /etc/tmpfiles.d 0755; apex_atomic_input ' + cleanup_policy, cleanup)
+            ssh.run(remote_atomic_input(stage + '/bundle.tar'), bundle)
+            ssh.run(remote_atomic_input(stage + '/user-data'), source.read_bytes())
+            install = remote_bundle_install(stage, installed, digest)
+            progress('Installing the versioned target runtime and validating enrollment input.')
+            ssh.run(install)
+            base = ['python3', '-m', 'engine.provisioning.target']
+            options = ['--user-data', stage + '/user-data', '--revision', revision, '--bundle-digest', digest]
+            # State remains inspectable after disconnect; user-data is removed by the durable unit.
+            enrollment = shlex.join(base + ['enroll'] + options + ['--heartbeat', stage + '/heartbeat'])
+            validation = shlex.join(base + ['validate'] + options)
+            access_preflight = shlex.join(['python3', '-m', 'engine.provisioning.access', 'preflight'])
+            seed = shlex.join(['bash', installed + '/engine/provisioning/seed.sh'])
+            wrapper = ('umask 077; cd ' + installed + '; '
+                       'trap ' + shlex.quote('rm -f ' + stage + '/user-data ' + stage + '/heartbeat ' + cleanup_policy) + ' EXIT; '
+                       '{ ' + seed + ' && ' + validation + ' && ' + access_preflight + ' && ' + enrollment + '; } >' + stage + '/runner.log 2>&1; rc=$?; '
+                       'printf "%s\\n" "$rc" >' + stage + '/result.new; mv ' + stage + '/result.new ' + stage + '/result; exit "$rc"')
+            unit = 'apex-init-' + run_id
+            record = json.dumps({'unit': unit, 'staging': stage, 'revision': revision, 'bundle_digest': digest}).encode()
+            ssh.run(remote_atomic_input('/var/lib/apex-init/remote.json'), record)
+            ssh.run('touch ' + stage + '/heartbeat; ' + shlex.join(['systemd-run', '--quiet', '--unit', unit,
+                    '--property=Type=exec', '--property=KillMode=process', '/bin/sh', '-c', wrapper]))
+            launched = True
+            progress('Enrollment is running under ' + unit + '; target staging: ' + stage)
+            while True:
+                status = ssh.run('if test -f ' + stage + '/result; then cat ' + stage + '/result; '
+                                 'elif systemctl is-active --quiet ' + unit + '; then touch ' + stage + '/heartbeat; printf running; '
+                                 'else printf stopped; fi').stdout.decode().strip()
+                if status != 'running':
+                    finished = True
+                    if status != '0':
+                        raise TransportError('Target provisioning failed; inspect root-owned ' + stage +
+                                             '/runner.log and /var/lib/apex-init/enrollment.json')
+                    break
+                time.sleep(5)
+            progress('Enrollment verified. Checking protected administrator SSH handover.')
+
+            def access(operation, token, original_port, address, fresh):
+                if operation == 'status':
+                    result = ssh.run(REMOTE_PATH_GUARD + '\napex_guard_file /var/lib/apex-init/access/state.json; '
+                                     'cat /var/lib/apex-init/access/state.json', fresh=True)
+                    return json.loads(result.stdout)
+                args = ['python3', '-m', 'engine.provisioning.access', operation, '--token', token]
+                if original_port:
+                    args += ['--original-port', str(original_port), '--client-address', address]
+                result = ssh.run('cd ' + installed + ' && ' + shlex.join(args), fresh=fresh)
+                return json.loads(result.stdout)
+
+            protected_handover(ssh, access, run_id, int(connection[3]), connection[0])
+            access_committed = True
+            result = ssh.run('cd ' + installed + ' && ' + shlex.join(base + ['complete']), fresh=True)
+            completed = json.loads(result.stdout)
+            progress('Initialization and fresh administrator access verified; provisioner ' + revision + ', bundle ' + digest)
+            repository = completed.get('stages', {}).get('repository', {}).get('result', {})
+            if repository.get('commit'):
+                progress('Repository ' + repository['commit'] + '; commons ' + repository.get('commons_commit', 'unknown'))
+            return completed
+        except HandoverUncertain:
+            access_uncertain = True
+            raise
+        finally:
+            if stage and (not launched or finished):
+                for fresh in (True, False):
+                    try:
+                        ssh.run('rm -f ' + stage + '/user-data ' + stage + '/heartbeat ' + stage + '/bundle.tar ' + cleanup_policy, fresh=fresh)
+                        break
+                    except Exception:
+                        pass
+            if launched and not finished:
+                progress('Coordinator stopped. The current target stage may finish; later stages stop after heartbeat expiry.')
+            if access_uncertain:
+                progress('Access commit outcome is unknown; verify which endpoint is active before resuming. Rollback may already be disabled.')
+                endpoints = [('Administrator candidate', 'adam', 2222), ('Bootstrap candidate', ssh.user, ssh.resolved_port)]
+            else:
+                label = 'Safe resume' + ('' if access_committed else ' after any pending rollback')
+                endpoints = [(label, 'adam' if access_committed else ssh.user, 2222 if access_committed else ssh.resolved_port)]
+            for label, resume_user, resume_port in endpoints:
+                resume = './apex init ' + resume_user + '@' + ssh.host + ' --port ' + str(resume_port) + ' --user-data ' + shlex.quote(str(source))
+                if ssh.identity:
+                    resume += ' --identity-file ' + shlex.quote(str(ssh.identity))
+                progress(label + ': ' + resume)

ci/scripts/init-node/provisioning/userdata.py

@@ -0,0 +1,29 @@
+"""Target-only decoding; importing the CLI never requires a YAML dependency."""
+from .contract import ContractError, normalize_user_data
+
+
+def decode_user_data(content):
+    import yaml
+
+    class Loader(yaml.SafeLoader):
+        def compose_node(self, parent, index):
+            if self.check_event(yaml.AliasEvent):
+                raise ContractError('YAML aliases are not supported')
+            return super().compose_node(parent, index)
+
+        def construct_mapping(self, node, deep=False):
+            result = {}
+            for key_node, value_node in node.value:
+                key = self.construct_object(key_node, deep=deep)
+                if not isinstance(key, str) or key in result:
+                    raise ContractError('YAML mapping keys must be unique strings')
+                result[key] = self.construct_object(value_node, deep=deep)
+            return result
+
+    if len(content) > 1024 * 1024:
+        raise ContractError('user-data exceeds the supported size')
+    try:
+        document = yaml.load(content, Loader=Loader)
+    except (yaml.YAMLError, UnicodeError, RecursionError):
+        raise ContractError('invalid or unsupported YAML user-data') from None
+    return normalize_user_data(document)