Remove the image build and consolidate operational code under scripts

scripts/provisioning/__init__.py

@@ -0,0 +1 @@
+"""Host-side Docker installation and SSH handover."""

scripts/provisioning/access.py

@@ -0,0 +1,597 @@
+"""Protected SSH handover with a separately locked, persistent rollback timer."""
+from __future__ import annotations
+
+import argparse
+import base64
+import fcntl
+import fnmatch
+import hashlib
+import ipaddress
+import json
+import os
+from pathlib import Path
+import pwd
+import re
+import shlex
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+
+
+class AccessError(RuntimeError):
+    pass
+
+
+def include_arguments(value):
+    arguments, word, quote, started = [], [], None, False
+    for character in value:
+        if character == '\\':
+            raise AccessError('Escaped SSH Include paths require manual reconciliation')
+        if quote:
+            if character == quote:
+                quote = None
+            else:
+                word.append(character)
+        elif character in ('"', "'"):
+            quote, started = character, True
+        elif character in ' \t\r':
+            if started:
+                arguments.append(''.join(word))
+                word, started = [], False
+        elif character == '#' and not started:
+            break
+        else:
+            word.append(character)
+            started = True
+    if quote:
+        raise AccessError('Unclosed SSH Include quote requires manual reconciliation')
+    if started:
+        arguments.append(''.join(word))
+    if not arguments or len(arguments) > 128 or any(not item or len(item) > 4096 for item in arguments):
+        raise AccessError('Unsupported SSH Include paths require manual reconciliation')
+    return arguments
+
+
+MANUAL_PROFILES = ('debian', 'proxmox')
+OPERATOR_MATCH = """Match User adam
+AuthenticationMethods publickey
+PubkeyAuthentication yes
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+PermitEmptyPasswords no
+X11Forwarding no""".splitlines()
+
+
+class Host:
+    def __init__(self, root=Path('/'), runner=subprocess.run, profile=None):
+        if profile is not None and profile not in MANUAL_PROFILES:
+            raise AccessError('Unknown SSH access profile')
+        self.profile = profile
+        self.root = Path(root).resolve()
+        self.runner = runner
+
+    def path(self, path):
+        return self.root / path.lstrip('/')
+
+    def command(self, args, check=True):
+        result = self.runner(args, capture_output=True, text=True, timeout=60)
+        if check and result.returncode:
+            raise AccessError('Access command failed: ' + args[0])
+        return result
+
+    def boot_id(self):
+        return self.path('/proc/sys/kernel/random/boot_id').read_text().strip()
+
+    def fingerprint(self):
+        digest = hashlib.sha256()
+        paths = [self.path('/etc/ssh/sshd_config'), self.path('/etc/default/ssh')]
+        for directory in ('/etc/ssh/sshd_config.d', '/etc/systemd/system/ssh.service.d',
+                          '/run/systemd/system/ssh.service.d', '/usr/lib/systemd/system/ssh.service.d',
+                          '/lib/systemd/system/ssh.service.d'):
+            paths += sorted(self.path(directory).glob('*.conf'))
+        for path in paths:
+            digest.update(str(path.relative_to(self.root)).encode())
+            digest.update(path.read_bytes() if path.exists() else b'<absent>')
+        return digest.hexdigest()
+
+    def unconditional_configuration(self):
+        remaining, directory_entries, visited = 1024 * 1024, 0, set()
+
+        def expand(pattern):
+            nonlocal directory_entries
+            if any(token in pattern for token in ('[:', '[.', '[=', '[^')):
+                raise AccessError('Extended SSH Include glob syntax requires manual reconciliation')
+            absolute = pattern if pattern.startswith('/') else '/etc/ssh/' + pattern
+            mapped = self.path(absolute)
+            if len(mapped.parts) > 64:
+                raise AccessError('SSH Include path exceeds supported limits; manual reconciliation required')
+            candidates = [Path(mapped.anchor)]
+            for component in mapped.parts[1:]:
+                expanded = []
+                for parent in candidates:
+                    if not any(character in component for character in '*?['):
+                        candidate = parent / component
+                        if os.path.lexists(candidate):
+                            expanded.append(candidate)
+                        continue
+                    try:
+                        with os.scandir(parent) as entries:
+                            for entry in entries:
+                                directory_entries += 1
+                                if directory_entries > 4096:
+                                    raise AccessError('SSH Include traversal exceeds supported limits; manual reconciliation required')
+                                if entry.name.startswith('.') and not component.startswith('.'):
+                                    continue
+                                if fnmatch.fnmatchcase(entry.name, component):
+                                    expanded.append(Path(entry.path))
+                    except (FileNotFoundError, NotADirectoryError):
+                        continue
+                if len(expanded) > 128:
+                    raise AccessError('SSH Include expansion exceeds supported limits; manual reconciliation required')
+                candidates = expanded
+            return sorted(candidates)
+
+        def inspect(path, ancestors):
+            nonlocal remaining
+            information = path.stat()
+            identity = (information.st_dev, information.st_ino)
+            if identity in ancestors or len(ancestors) >= 16:
+                raise AccessError('Recursive SSH Include configuration requires manual reconciliation')
+            if identity in visited:
+                return
+            if len(visited) >= 128 or not stat.S_ISREG(information.st_mode) or information.st_size > remaining:
+                raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required')
+            visited.add(identity)
+            with path.open('rb') as stream:
+                content = stream.read(remaining + 1)
+            remaining -= len(content)
+            if remaining < 0:
+                raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required')
+            lines = content.decode().splitlines()
+            if self.profile and path == self.path('/etc/ssh/sshd_config'):
+                # The legacy action appended this exact terminal block. Any additional
+                # directives would change its meaning and require manual inspection.
+                marker = '# APEX operator SSH policy'
+                if marker in lines:
+                    position = lines.index(marker)
+                    if [item.strip() for item in lines[position + 1:] if item.strip()] == OPERATOR_MATCH:
+                        lines = lines[:position]
+            for line in lines:
+                line = line.lstrip()
+                if not line or line.startswith('#'):
+                    continue
+                if line.startswith('"'):
+                    keyword, separator, value = line[1:].partition('"')
+                    if not separator:
+                        raise AccessError('Unsupported SSH keyword quoting requires manual reconciliation')
+                else:
+                    parsed = re.match(r'([^\s=]+)(.*)', line)
+                    if not parsed:
+                        raise AccessError('Unsupported SSH configuration syntax requires manual reconciliation')
+                    keyword, value = parsed.groups()
+                    value = value.lstrip()
+                    if value.startswith('='):
+                        value = value[1:]
+                if keyword.lower() == 'match':
+                    raise AccessError('SSH Match directives require manual reconciliation before initialization')
+                if keyword.lower() != 'include':
+                    continue
+                for pattern in include_arguments(value):
+                    for included in expand(pattern):
+                        inspect(included, ancestors | {identity})
+
+        try:
+            inspect(self.path('/etc/ssh/sshd_config'), set())
+        except (OSError, UnicodeError) as exc:
+            raise AccessError('Cannot inspect SSH configuration and includes; manual reconciliation required') from exc
+
+    def bootstrap_preflight(self):
+        for activity in ('is-active', 'is-enabled'):
+            if self.command(['systemctl', activity, '--quiet', 'ssh.socket'], False).returncode == 0:
+                raise AccessError('SSH socket activation requires manual handover; no access changes made')
+        if self.command(['systemctl', 'is-active', '--quiet', 'ssh.service'], False).returncode:
+            raise AccessError('An active ssh.service is required')
+        mode = self.command(['systemctl', 'show', 'ssh.service', '-p', 'KillMode', '--value']).stdout.strip()
+        if mode != 'process':
+            raise AccessError('ssh.service must preserve established sessions with KillMode=process')
+        arguments = self.service_arguments()
+        managed = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        if managed.exists():
+            commands = [line.removeprefix('ExecStart=') for line in managed.read_text().splitlines()
+                        if line.startswith('ExecStart=') and line != 'ExecStart=']
+            if len(commands) != 1 or arguments != shlex.split(commands[0]):
+                raise AccessError('Active SSH service configuration differs from its managed invocation')
+        elif arguments != ['/usr/sbin/sshd', '-D']:
+            raise AccessError('Custom SSH service invocation requires manual access reconciliation')
+        defaults = self.path('/etc/default/ssh')
+        if defaults.exists():
+            for line in defaults.read_text().splitlines():
+                if re.match(r'^\s*SSHD_OPTS\s*=', line):
+                    value = line.split('=', 1)[1].strip().strip('\"\'')
+                    if value:
+                        raise AccessError('Custom SSHD_OPTS require manual access reconciliation')
+        self.unconditional_configuration()
+        self.command(['/usr/sbin/sshd', '-t'])
+        self.live_arguments()
+
+    def preflight(self):
+        self.bootstrap_preflight()
+        self.command(['visudo', '-c'])
+        self.command(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true'])
+
+    def service_arguments(self):
+        output = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout
+        matches = re.findall(r'argv\[\]=([^;]+)', output)
+        if len(matches) != 1:
+            raise AccessError('Unable to verify the SSH service invocation')
+        arguments = shlex.split(matches[0])
+        # Debian's stock unit has this optional variable. Its actual expansion is
+        # independently checked against /proc below, so a nonempty value fails.
+        if arguments == ['/usr/sbin/sshd', '-D', '$SSHD_OPTS']:
+            arguments.pop()
+        if not arguments or arguments[0] != '/usr/sbin/sshd':
+            raise AccessError('SSH service does not invoke the supported daemon')
+        return arguments
+
+    def live_arguments(self):
+        expected = self.service_arguments()
+        pid = self.command(['systemctl', 'show', 'ssh.service', '-p', 'MainPID', '--value']).stdout.strip()
+        if not pid.isdecimal() or int(pid) <= 1:
+            raise AccessError('SSH service has no verifiable active listener process')
+        process = self.path('/proc/' + pid)
+        try:
+            if os.readlink(process / 'exe') != '/usr/sbin/sshd':
+                raise AccessError('Active SSH process does not run the supported daemon')
+            parts = [part for part in (process / 'cmdline').read_bytes().decode().split('\0') if part]
+        except (OSError, UnicodeError):
+            raise AccessError('Unable to inspect the active SSH listener invocation') from None
+        if len(parts) == 1 and parts[0].startswith('sshd: ') and ' [listener]' in parts[0]:
+            actual = shlex.split(parts[0][6:].split(' [listener]', 1)[0])
+        else:
+            actual = parts
+        if actual != expected:
+            raise AccessError('Running SSH listener differs from the active service invocation')
+        return actual, pid
+
+    def verify_active(self, ports, final, address):
+        args, pid = self.live_arguments()
+        self.verify_policy([arg for arg in args if arg != '-D'], ports, final, address)
+        if self.listener_ports(pid) != set(ports):
+            raise AccessError('Active SSH listening ports differ from the declared policy')
+
+    def listener_ports(self, pid):
+        output = self.command(['ss', '-H', '-ltnp']).stdout
+        actual_ports = set()
+        for line in output.splitlines():
+            if '("sshd",' not in line:
+                continue
+            if set(re.findall(r'pid=(\d+)', line)) != {pid}:
+                raise AccessError('An additional SSH listener exists outside the managed service')
+            fields = line.split()
+            if len(fields) < 5 or not fields[3].rsplit(':', 1)[-1].isdecimal():
+                raise AccessError('Unable to verify the active SSH listening sockets')
+            actual_ports.add(int(fields[3].rsplit(':', 1)[-1]))
+        return actual_ports
+
+    @property
+    def final_ports(self):
+        return [22, 2222] if self.profile == 'proxmox' else [2222]
+
+    def authentication_options(self):
+        proxmox = self.profile == 'proxmox'
+        options = {'PermitRootLogin': 'prohibit-password' if proxmox else 'no',
+                   'PasswordAuthentication': 'no', 'KbdInteractiveAuthentication': 'no',
+                   'PubkeyAuthentication': 'yes', 'AuthenticationMethods': 'publickey'}
+        if not proxmox:
+            options['AllowUsers'] = 'adam'
+        options.update(X11Forwarding='no', PermitEmptyPasswords='no')
+        return options
+
+    def policy(self, ports, final, address):
+        args = ['/usr/sbin/sshd']
+        for port in sorted(set(ports)):
+            args += ['-p', str(port)]
+        if final:
+            for key, value in self.authentication_options().items():
+                args += ['-o', f'{key}={value}']
+        elif self.profile:
+            # Keep existing managed authentication options when adding the probe port.
+            current = iter(self.service_arguments()[1:])
+            for flag in current:
+                if flag == '-D':
+                    continue
+                value = next(current, '')
+                if flag == '-p' and value.isdecimal():
+                    continue
+                if flag != '-o' or not re.fullmatch(r'[A-Za-z]+=[A-Za-z0-9,-]+', value):
+                    raise AccessError('Cannot safely extend the existing SSH invocation')
+                args += [flag, value]
+        self.verify_policy(args, ports, final, address)
+        return args
+
+    def verify_policy(self, args, ports, final, address):
+        self.unconditional_configuration()
+        self.command(args + ['-t'])
+        users = {'adam', 'root'}
+        if final:
+            users.update(account.pw_name for account in pwd.getpwall())
+        for user in sorted(users):
+            output = self.command(args + ['-T', '-C', f'user={user},host=localhost,addr={address}']).stdout
+            values = {}
+            for line in output.splitlines():
+                key, _, value = line.partition(' ')
+                values.setdefault(key, []).append(value)
+            if set(values.get('port', [])) != {str(p) for p in ports}:
+                raise AccessError('Effective SSH ports do not match the handover policy')
+            if final:
+                expected = {key.lower(): [value] for key, value in self.authentication_options().items()}
+                if self.profile == 'proxmox':
+                    if values.get('permitrootlogin') == ['without-password']:
+                        values['permitrootlogin'] = ['prohibit-password']
+                    if any(key in values for key in ('allowusers', 'denyusers', 'allowgroups', 'denygroups')):
+                        raise AccessError('SSH account restrictions conflict with Proxmox root-key access')
+                if any(values.get(key) != value for key, value in expected.items()):
+                    raise AccessError('Effective SSH authentication policy is inconsistent')
+
+    def install_policy(self, ports, final, address):
+        args = self.policy(ports, final, address)
+        path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        atomic(path, ('[Service]\nExecStart=\nExecStart=' + ' '.join(args + ['-D']) + '\n').encode(), 0o644)
+        self.command(['systemctl', 'daemon-reload'])
+        actual = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout
+        match = re.search(r'argv\[\]=([^;]+)', actual)
+        if not match or shlex.split(match[1]) != args + ['-D']:
+            raise AccessError('Another service override prevents the managed SSH policy')
+        self.command(['systemctl', 'restart', 'ssh.service'])
+        self.verify_active(ports, final, address)
+
+    def arm(self):
+        bundle = Path(__file__).resolve().parents[1]
+        if not re.fullmatch(r'/usr/local/lib/apex-provisioner/[a-f0-9]{64}', str(bundle)):
+            raise AccessError('Access runner must execute from the installed root-owned bundle')
+        service = ('[Unit]\nDescription=APEX pending access rollback\nAfter=local-fs.target\n'
+                   '[Service]\nType=oneshot\nWorkingDirectory=' + str(bundle) + '\n'
+                   'ExecStart=/usr/bin/python3 -m provisioning.access tick\n')
+        timer = ('[Unit]\nDescription=APEX access rollback deadline and boot recovery\n'
+                 '[Timer]\nOnBootSec=5s\nOnUnitActiveSec=10s\nAccuracySec=1s\n'
+                 '[Install]\nWantedBy=timers.target\n')
+        atomic(self.path('/etc/systemd/system/apex-access-rollback.service'), service.encode(), 0o644)
+        atomic(self.path('/etc/systemd/system/apex-access-rollback.timer'), timer.encode(), 0o644)
+        self.command(['systemctl', 'daemon-reload'])
+        self.command(['systemctl', 'enable', '--now', 'apex-access-rollback.timer'])
+
+    def disarm(self):
+        self.command(['systemctl', 'disable', '--now', 'apex-access-rollback.timer'])
+
+    def restore(self, state):
+        path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        if state['previous'] is None:
+            path.unlink(missing_ok=True)
+        else:
+            atomic(path, base64.b64decode(state['previous']), 0o644)
+        self.command(['/usr/sbin/sshd', '-t'])
+        self.command(['systemctl', 'daemon-reload'])
+        self.command(['systemctl', 'restart', 'ssh.service'])
+
+
+def atomic(path, data, mode=0o600):
+    path = Path(path)
+    for parent in [path, *path.parents]:
+        if parent.is_symlink():
+            raise AccessError('Refusing a symlink in a managed access path')
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix='.apex-access-', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'wb') as stream:
+            os.fchmod(stream.fileno(), mode)
+            stream.write(data)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+    directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(directory)
+    finally:
+        os.close(directory)
+
+
+class Transition:
+    def __init__(self, host=None, now=time.monotonic, profile=None):
+        if profile is not None and profile not in MANUAL_PROFILES:
+            raise AccessError('Unknown SSH access profile')
+        self.profile = profile
+        self.host = host or Host()
+        self.host.profile = profile
+        self.now = now
+        self.directory = self.host.path('/var/lib/apex-init/access')
+
+    def load(self):
+        path = self.directory / 'manual-state.json'
+        return json.loads(path.read_text()) if path.exists() else None
+
+    def save(self, state):
+        atomic(self.directory / 'manual-state.json', json.dumps(state, sort_keys=True).encode())
+
+    def execute(self, operation, token=None, original_port=None, client_address=None):
+        # The watchdog must recover SSH even when another APEX action holds init.lock.
+        if operation in ('tick', 'rollback'):
+            return self._execute(operation, token, original_port, client_address)
+        from .state import ProvisionError, exclusive_lock
+        try:
+            with exclusive_lock(self.host.path('/var/lib/apex-init/init.lock')):
+                return self._execute(operation, token, original_port, client_address)
+        except ProvisionError as error:
+            raise AccessError(str(error)) from None
+
+    def _execute(self, operation, token=None, original_port=None, client_address=None):
+        if token is not None and not re.fullmatch(r'[a-f0-9]{32}', token):
+            raise AccessError('Invalid access transition token')
+        for path in [self.directory, *self.directory.parents]:
+            if path.is_symlink():
+                raise AccessError('Invalid access state directory')
+        self.directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+        if self.directory.stat().st_uid != os.geteuid():
+            raise AccessError('Access state directory is not owned by the provisioner user')
+        os.chmod(self.directory, 0o700)
+        descriptor = os.open(self.directory / 'lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+        with os.fdopen(descriptor, 'w') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            state = self.load()
+            active = ('transition', 'final', 'recovery_pending')
+            # A historical bundle owns its pending rollback and the shared timer.
+            legacy_path = self.directory / 'state.json'
+            if legacy_path.exists():
+                legacy = json.loads(legacy_path.read_text())
+                if legacy['status'] in active or legacy.get('disarm_pending'):
+                    raise AccessError('Another access workflow is pending; wait for rollback before retrying')
+            if state and self.profile and state.get('profile') != self.profile and operation != 'tick':
+                if operation != 'begin' or state['status'] in active:
+                    raise AccessError('Access transition belongs to a different platform')
+                state = None
+            if state and state.get('disarm_pending') and state['status'] in ('committed', 'rolled_back'):
+                self._disarm(state)
+            if (state and token and token == state.get('token')
+                    and (operation, state['status']) in (('commit', 'committed'), ('rollback', 'rolled_back'))):
+                return {'status': state['status']}
+            if state and state['status'] == 'recovery_pending':
+                self._rollback(state)
+                if operation != 'tick':
+                    raise AccessError('Pending recovery completed; retry initialization')
+                return {'status': 'rolled_back'}
+            pending = state and state['status'] in ('transition', 'final')
+            if pending and (self.now() >= state['deadline'] or self.host.boot_id() != state['boot_id']):
+                self._rollback(state)
+                if operation != 'tick':
+                    raise AccessError('Uncommitted access transition expired or rebooted; previous access restored')
+                return {'status': 'rolled_back'}
+            if operation == 'tick':
+                return {'status': state['status'] if state else 'absent'}
+            if operation == 'begin':
+                if pending:
+                    raise AccessError('A pending access transition exists; wait for rollback before retrying')
+                if not token or not original_port or not 1 <= original_port <= 65535:
+                    raise AccessError('Access transition requires an original port and token')
+                try:
+                    ipaddress.ip_address(client_address)
+                except ValueError:
+                    raise AccessError('Invalid SSH client address') from None
+                self.host.preflight()
+                path = self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+                return self._begin_manual(token, original_port, client_address, path)
+            if not pending or token != state['token']:
+                raise AccessError('No matching pending access transition')
+            if operation == 'rollback':
+                self._rollback(state)
+                return {'status': 'rolled_back'}
+            if state.get('fingerprint') != self.host.fingerprint():
+                raise AccessError('External SSH configuration changed; finalization stopped, rollback remains armed')
+            if operation == 'finalize':
+                if state['status'] != 'transition':
+                    raise AccessError('Access transition is not ready for final policy')
+                try:
+                    if not state.get('preserve_policy'):
+                        state['ssh_changed'] = True
+                        self.save(state)
+                        self.host.install_policy(self.host.final_ports, True, state['client_address'])
+                    state['authenticated_after_us'] = time.monotonic_ns() // 1000
+                    state['status'] = 'final'
+                    state['fingerprint'] = self.host.fingerprint()
+                    self.save(state)
+                except Exception:
+                    self._rollback(state)
+                    raise
+                return {'status': 'final'}
+            if operation == 'commit' and state['status'] == 'final':
+                self.host.verify_active(self.host.final_ports, True, state['client_address'])
+                state['status'] = 'committed'
+                state['disarm_pending'] = True
+                self.save(state)  # Timer observes this before it can restore anything.
+                self._disarm(state)
+                return {'status': 'committed'}
+            raise AccessError('Invalid access transition operation')
+
+    def _begin_manual(self, token, original_port, address, path):
+        _, pid = self.host.live_arguments()
+        ports = self.host.listener_ports(pid)
+        if original_port not in ports:
+            raise AccessError('Original SSH port is not active on the managed daemon')
+        preserve_policy = False
+        if ports == set(self.host.final_ports):
+            try:
+                self.host.verify_active(self.host.final_ports, True, address)
+                preserve_policy = True
+            except AccessError:
+                pass
+        previous = base64.b64encode(path.read_bytes()).decode() if path.exists() else None
+        state = {'status': 'transition', 'profile': self.profile, 'token': token,
+                 'deadline': self.now() + 300, 'boot_id': self.host.boot_id(),
+                 'bundle': str(Path(__file__).resolve().parents[1]),
+                 'client_address': address, 'original_port': original_port, 'previous': previous,
+                 'preserve_policy': preserve_policy, 'ssh_changed': 2222 not in ports}
+        self.host.arm()
+        self.save(state)
+        try:
+            if 2222 not in ports:
+                self.host.install_policy(sorted(ports | {2222}), False, address)
+            state['fingerprint'] = self.host.fingerprint()
+            state['authenticated_after_us'] = time.monotonic_ns() // 1000
+            self.save(state)
+        except Exception:
+            self._rollback(state)
+            raise
+        return {'status': 'transition'}
+
+    def _rollback(self, state):
+        errors = []
+        if not state.get('ssh_restored') and state.get('ssh_changed', True):
+            try:
+                self.host.restore(state)
+                state['ssh_restored'] = True
+            except Exception:
+                errors.append('ssh')
+        if errors:
+            state['status'] = 'recovery_pending'
+            state['recovery_errors'] = errors
+            self.save(state)
+            raise AccessError('Recovery remains pending for ' + ', '.join(errors) +
+                              '; watchdog remains armed; inspect the root-owned access record')
+        state['status'] = 'rolled_back'
+        state.pop('recovery_errors', None)
+        state['disarm_pending'] = True
+        self.save(state)
+        self._disarm(state)
+
+    def _disarm(self, state):
+        self.host.disarm()
+        state.pop('disarm_pending', None)
+        self.save(state)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('operation', choices=['begin', 'finalize', 'commit', 'rollback', 'tick'])
+    parser.add_argument('--profile', choices=MANUAL_PROFILES)
+    parser.add_argument('--token')
+    parser.add_argument('--original-port', type=int)
+    parser.add_argument('--client-address')
+    args = parser.parse_args(argv)
+    if os.geteuid() != 0:
+        parser.error('Access handover requires root')
+    try:
+        if args.operation != 'tick' and not args.profile:
+            parser.error('A host profile is required')
+        result = Transition(profile=args.profile).execute(args.operation, args.token, args.original_port, args.client_address)
+        print(json.dumps(result))
+        return 0
+    except AccessError as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/provisioning/docker.py

@@ -0,0 +1,301 @@
+"""Verify or install a local Docker Engine and CLI plugins."""
+from __future__ import annotations
+
+from contextlib import contextmanager
+import fcntl
+from functools import wraps
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import tempfile
+import urllib.request
+
+
+class DockerError(RuntimeError):
+    """A required Docker postcondition could not be established."""
+
+
+def expected_errors(operation):
+    @wraps(operation)
+    def guarded(*args, **kwargs):
+        try:
+            return operation(*args, **kwargs)
+        except (OSError, ValueError) as exc:
+            raise DockerError("Docker filesystem or saved state is invalid; inspect the target before retrying.") from exc
+    return guarded
+
+
+DOCKER_PACKAGES = ("docker-ce", "docker-ce-cli", "containerd.io",
+                   "docker-buildx-plugin", "docker-compose-plugin")
+DOCKER_CONFLICTS = ("docker.io", "docker-doc", "docker-compose", "docker-buildx",
+                    "podman-docker", "containerd", "runc")
+DOCKER_CONFIG = {"data-root": "/srv/docker", "log-driver": "json-file",
+                 "log-opts": {"max-size": "20m", "max-file": "3"}}
+POLICY = Path("/usr/sbin/policy-rc.d")
+POLICY_STATE = Path("/usr/sbin/.apex-policy-rc.d")
+APT = ["apt-get", "-o", "DPkg::Lock::Timeout=60", "-o",
+       "Acquire::Retries=2", "-o", "Acquire::http::Timeout=30", "-o",
+       "Acquire::https::Timeout=30"]
+
+
+def run(args, *, check=True, timeout=300):
+    try:
+        result = subprocess.run(args, text=True, capture_output=True, timeout=timeout,
+                                env={**os.environ, "LC_ALL": "C", "DEBIAN_FRONTEND": "noninteractive"})
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        raise DockerError(f"Cannot complete {args[0]}: {exc}") from exc
+    if check and result.returncode:
+        raise DockerError(f"{args[0]} failed with exit status {result.returncode}; inspect target system logs.")
+    return result
+
+
+def package_versions():
+    architecture = run(["dpkg", "--print-architecture"]).stdout.strip()
+    result = run(["dpkg-query", "-W", "-f=${binary:Package}\t${Status}\t${Version}\n"])
+    installed = {}
+    for line in result.stdout.splitlines():
+        fields = line.split("\t")
+        if len(fields) == 3 and fields[1].split()[1:] == ["ok", "installed"]:
+            installed[fields[0].removesuffix(":" + architecture)] = fields[2]
+    return installed
+
+
+def write_managed(path, content, mode=0o644):
+    path = Path(path)
+    for parent in (path, *path.parents):
+        if parent.is_symlink():
+            raise DockerError(f"Refusing managed symlink: {parent}")
+    payload = content.encode() if isinstance(content, str) else content
+    if path.exists() and path.read_bytes() == payload:
+        if path.stat().st_mode & 0o777 != mode:
+            path.chmod(mode)
+        return False
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=path.parent)
+    try:
+        with os.fdopen(fd, "wb") as stream:
+            stream.write(payload)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(temporary, mode)
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+    return True
+
+
+def restore_service_policy():
+    if not POLICY_STATE.exists():
+        return
+    backup = POLICY_STATE / "original"
+    if (POLICY_STATE.is_symlink() or POLICY_STATE.stat().st_uid != os.geteuid()):
+        raise DockerError("Refusing untrusted service policy recovery state.")
+    if (POLICY_STATE / "ready").exists() and POLICY.exists():
+        original_matches = (backup.exists() and not POLICY.is_symlink() and
+                            POLICY.read_bytes() == backup.read_bytes())
+        symlink_matches = (backup.is_symlink() and POLICY.is_symlink() and
+                           os.readlink(backup) == os.readlink(POLICY))
+        guard_matches = not POLICY.is_symlink() and POLICY.read_bytes() == b"#!/bin/sh\nexit 101\n"
+        if not (original_matches or symlink_matches or guard_matches):
+            raise DockerError("Service policy changed outside APEX; recovery requires explicit reconciliation.")
+    if backup.exists() or backup.is_symlink():
+        # Copy first: an interrupted restoration must retain its recovery copy.
+        temporary = POLICY_STATE / "restoring"
+        if temporary.exists() or temporary.is_symlink():
+            temporary.unlink()
+        shutil.copy2(backup, temporary, follow_symlinks=False)
+        os.replace(temporary, POLICY)
+    elif (POLICY_STATE / "absent").exists():
+        POLICY.unlink(missing_ok=True)
+    elif (POLICY_STATE / "ready").exists():
+        raise DockerError("Service policy recovery is incomplete; inspect /usr/sbin/.apex-policy-rc.d.")
+    shutil.rmtree(POLICY_STATE)
+
+
+@contextmanager
+def service_start_guard():
+    lock_path = Path("/run/lock/apex-package-policy.lock")
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a") as lock:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise DockerError("Another APEX package operation is running.") from None
+        with _service_start_guard():
+            yield
+
+
+@contextmanager
+def _service_start_guard():
+    restore_service_policy()
+    POLICY_STATE.mkdir(mode=0o700)
+    try:
+        if POLICY.exists() or POLICY.is_symlink():
+            shutil.copy2(POLICY, POLICY_STATE / "original", follow_symlinks=False)
+        else:
+            (POLICY_STATE / "absent").touch()
+        (POLICY_STATE / "ready").touch()
+        # Replace the path itself, preserving any original symlink in the backup.
+        temporary = POLICY_STATE / "guard"
+        temporary.write_text("#!/bin/sh\nexit 101\n")
+        temporary.chmod(0o755)
+        os.replace(temporary, POLICY)
+        yield
+    finally:
+        restore_service_policy()
+
+
+def install_missing(packages):
+    installed = package_versions()
+    missing = [package for package in packages if package not in installed]
+    if not missing:
+        return []
+    run([*APT, "update", "--error-on=any"])
+    command = [*APT, "install", "-y", "--no-install-recommends", "--no-upgrade", "--no-remove", *missing]
+    plan = run([*command, "--simulate"]).stdout
+    if any(line.startswith("Remv ") or re.match(r"^Inst \S+ \[", line)
+           for line in plan.splitlines()):
+        raise DockerError("Missing packages require an upgrade/removal; reconcile packages explicitly before retrying.")
+    # Let dpkg finish before restoring the service-start policy.
+    run(command, timeout=None)
+    remaining = set(missing) - package_versions().keys()
+    if remaining:
+        raise DockerError("Required packages not installed: " + ", ".join(sorted(remaining)))
+    return missing
+
+
+def docker_preflight(installed):
+    conflicts = sorted(set(DOCKER_CONFLICTS) & installed.keys())
+    if conflicts:
+        raise DockerError("Conflicting Docker packages require explicit reconciliation: " + ", ".join(conflicts))
+    path = Path("/etc/docker/daemon.json")
+    try:
+        config = json.loads(path.read_text()) if path.exists() else {}
+    except (OSError, ValueError) as exc:
+        raise DockerError("Cannot validate existing Docker daemon configuration.") from exc
+    if not isinstance(config, dict):
+        raise DockerError("Docker daemon configuration must be an object.")
+    for name, value in DOCKER_CONFIG.items():
+        if name in config and config[name] != value:
+            raise DockerError(f"Conflicting Docker {name}; reconcile explicitly before initialization.")
+    old_data = Path("/var/lib/docker")
+    if old_data.exists() and any(old_data.iterdir()):
+        raise DockerError("Existing /var/lib/docker data must be reconciled explicitly; no data migration is automatic.")
+    data = Path("/srv/docker")
+    if data.is_symlink() or data.parent.is_symlink():
+        raise DockerError("Refusing a redirected Docker data-root.")
+    if data.exists() and any(data.iterdir()) and config.get("data-root") != "/srv/docker":
+        raise DockerError("Unmanaged /srv/docker data requires explicit reconciliation.")
+    if "docker-ce" in installed:
+        active = run(["systemctl", "is-active", "--quiet", "docker"], check=False).returncode == 0
+        if active:
+            root = run(["docker", "info", "--format", "{{.DockerRootDir}}"] ).stdout.strip()
+            if root != "/srv/docker" or any(config.get(key) != value for key, value in DOCKER_CONFIG.items()):
+                raise DockerError("Running Docker has incompatible data-root/log policy; reconcile it before retrying.")
+    return {**config, **DOCKER_CONFIG}
+
+
+DOCKER_KEY_SHA256 = "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570"
+LOCAL_DOCKER = ["docker", "--host", "unix:///var/run/docker.sock"]
+
+
+def _docker_source(arch):
+    key = Path("/etc/apt/keyrings/apex-docker.asc")
+    source = Path("/etc/apt/sources.list.d/apex-docker.sources")
+    content = ("Types: deb\nURIs: https://download.docker.com/linux/debian\nSuites: trixie\n"
+               f"Components: stable\nArchitectures: {arch}\nSigned-By: {key}\n")
+    for path in (key, source):
+        if any(part.is_symlink() for part in (path, *path.parents)):
+            raise DockerError(f"Refusing redirected APT path: {path}")
+    if source.exists() and source.read_text() != content:
+        raise DockerError(f"Reconcile existing Docker repository configuration: {source}")
+    if key.exists():
+        data = key.read_bytes()
+    else:
+        url = "https://download.docker.com/linux/debian/gpg"
+        with urllib.request.urlopen(url, timeout=30) as response:  # noqa: S310
+            data = response.read(1024 * 1024)
+    if hashlib.sha256(data).hexdigest() != DOCKER_KEY_SHA256:
+        raise DockerError("Docker repository signing key differs from the pinned release key.")
+    write_managed(key, data)
+    write_managed(source, content)
+
+
+def _docker_capabilities(root, *, complete_plugins=False):
+    capabilities = {"docker_data_root": root}
+    plugins = {"docker-compose-plugin": ("compose", ["compose", "version", "--short"]),
+               "docker-buildx-plugin": ("buildx", ["buildx", "version"])}
+    missing = []
+    for package, (name, args) in plugins.items():
+        result = run([*LOCAL_DOCKER, *args], check=False)
+        if result.returncode or not result.stdout.strip():
+            missing.append(package)
+        else:
+            capabilities[name] = result.stdout.strip()
+    if missing:
+        if complete_plugins and "docker-ce" in package_versions():
+            with service_start_guard():
+                install_missing(missing)
+            return _docker_capabilities(root)
+        raise DockerError("Existing Docker needs functional plugins: " + ", ".join(missing))
+    if not root:
+        raise DockerError("Unable to verify the local Docker data-root.")
+    return capabilities
+
+
+@expected_errors
+def apply_docker_policy():
+    if Path("/usr/bin/pveversion").exists():
+        raise DockerError("Docker installation on a Proxmox host requires separate administration.")
+    release = dict(line.split("=", 1) for line in Path("/etc/os-release").read_text().splitlines()
+                   if "=" in line and not line.startswith("#"))
+    arch = run(["dpkg", "--print-architecture"]).stdout.strip()
+    if (release.get("ID", "").strip('"') != "debian" or
+            release.get("VERSION_ID", "").strip('"') != "13" or arch not in ("amd64", "arm64")):
+        raise DockerError("Docker installation supports stock Debian 13 amd64 or arm64.")
+    info = [*LOCAL_DOCKER, "info", "--format", "{{.DockerRootDir}}"]
+    if shutil.which("docker"):
+        current = run(info, check=False)
+        if current.returncode == 0:
+            return _docker_capabilities(current.stdout.strip(), complete_plugins=True)
+    installed = package_versions()
+    source = Path("/etc/apt/sources.list.d/apex-docker.sources")
+    if (not source.exists() and
+            (shutil.which("docker") or set(DOCKER_PACKAGES).intersection(installed))):
+        raise DockerError("Existing Docker installation is inactive; reconcile it before retrying.")
+    config = docker_preflight(installed)
+    with service_start_guard():
+        install_missing(("ca-certificates",))
+        _docker_source(arch)
+        write_managed("/etc/docker/daemon.json", json.dumps(config, indent=2) + "\n")
+        install_missing(DOCKER_PACKAGES)
+    ensure_service("containerd")
+    ensure_service("docker")
+    root = run(info).stdout.strip()
+    if root != DOCKER_CONFIG["data-root"]:
+        raise DockerError("Docker started with an unexpected data-root; inspect its configuration.")
+    return _docker_capabilities(root)
+
+
+def ensure_service(name):
+    if run(["systemctl", "is-enabled", "--quiet", name], check=False).returncode:
+        run(["systemctl", "enable", name])
+    if run(["systemctl", "is-active", "--quiet", name], check=False).returncode:
+        run(["systemctl", "start", name])
+
+
+if __name__ == "__main__":
+    from .state import ProvisionError, exclusive_lock
+
+    try:
+        if os.geteuid() != 0 or not Path("/etc/debian_version").is_file():
+            raise DockerError("Docker setup requires root on Debian.")
+        with exclusive_lock("/var/lib/apex-init/init.lock"):
+            print(json.dumps(apply_docker_policy(), sort_keys=True))
+    except (DockerError, OSError, ProvisionError) as exc:
+        raise SystemExit(f"Docker setup failed: {exc}") from None

scripts/provisioning/manual_bundle.py

@@ -0,0 +1,76 @@
+"""Install an immutable watchdog bundle from the operator's checkout."""
+import hashlib
+import io
+import os
+from pathlib import Path
+import shutil
+import stat
+import tarfile
+import tempfile
+
+
+def _bundle_from_source(root):
+    root = Path(root).resolve()
+    files = [root / 'provisioning' / name for name in ('__init__.py', 'access.py', 'state.py')]
+    output = io.BytesIO()
+    with tarfile.open(fileobj=output, mode='w') as archive:
+        for path in files:
+            if (not path.is_file() or '__pycache__' in path.parts
+                    or any(part.is_symlink() for part in (path, *path.parents))):
+                raise RuntimeError('Provisioning bundle contains an unsafe source path')
+            data = path.read_bytes()
+            info = tarfile.TarInfo(path.relative_to(root).as_posix())
+            info.size, info.mode, info.mtime = len(data), 0o644, 0
+            info.uid = info.gid = 0
+            archive.addfile(info, io.BytesIO(data))
+    data = output.getvalue()
+    return data, hashlib.sha256(data).hexdigest()
+
+
+def _trusted(path):
+    info = path.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o022:
+        raise RuntimeError(f"Untrusted provisioning directory: {path}")
+
+
+def install(root=Path('/')):
+    source = Path(__file__).resolve().parents[1]
+    data, digest = _bundle_from_source(source)
+    # All ancestors from the filesystem root must be protected before publishing code.
+    parent = root
+    _trusted(parent)
+    for component in ('usr', 'local', 'lib', 'apex-provisioner'):
+        parent = parent / component
+        parent.mkdir(mode=0o755, exist_ok=True)
+        _trusted(parent)
+    destination = parent / digest
+    with tempfile.TemporaryDirectory(prefix='.install-', dir=parent) as temporary:
+        stage = Path(temporary) / 'bundle'
+        stage.mkdir(mode=0o755)
+        with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+            for member in archive:
+                if not member.isfile() or '..' in Path(member.name).parts or Path(member.name).is_absolute():
+                    raise RuntimeError('Invalid provisioning bundle member')
+                target = stage / member.name
+                target.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+                with archive.extractfile(member) as source_file, target.open('xb') as output:
+                    shutil.copyfileobj(source_file, output)
+                target.chmod(member.mode)
+        try:
+            os.rename(stage, destination)
+        except OSError:
+            _trusted(destination)
+            installed, existing_digest = _bundle_from_source(destination)
+            if existing_digest != digest or installed != data:
+                raise RuntimeError('Installed provisioning bundle differs from the checkout') from None
+            for path in destination.rglob('*'):
+                info = path.lstat()
+                if info.st_uid != os.geteuid() or info.st_mode & 0o022 or path.is_symlink():
+                    raise RuntimeError(f'Untrusted installed provisioning file: {path}') from None
+    return destination
+
+
+if __name__ == '__main__':
+    if os.geteuid() != 0:
+        raise SystemExit('Run bundle installation as root.')
+    print(install())

scripts/provisioning/state.py

@@ -0,0 +1,37 @@
+"""Serialize host configuration changes across provisioning commands."""
+import contextlib
+import fcntl
+import os
+from pathlib import Path
+import stat
+
+
+class ProvisionError(RuntimeError):
+    pass
+
+
+def safe_path(path):
+    path = Path(path).absolute()
+    for part in (*reversed(path.parents), path):
+        try:
+            info = part.lstat()
+        except FileNotFoundError:
+            continue
+        if stat.S_ISLNK(info.st_mode):
+            raise ProvisionError('managed path contains a symbolic link')
+    return path
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    path = safe_path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+    try:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise ProvisionError('another initialization is running; inspect status before retrying') from None
+        yield
+    finally:
+        os.close(fd)