Apply platform SSH policies through a recoverable port handover

README.md

@@ -121,7 +121,17 @@ apex config/adam --public-key-file /path/to/operator.pub
 apex repository/chown
 ```
 
-`config/adam` grants passwordless sudo. `repository/chown` gives adam ownership of the tracked checkout directly under `/srv` and its Git metadata. Open a new SSH connection as adam with the operator key and run `./commons/apex config/hardening` from the checkout in that session. An independent administrator login can also run the action after the operator’s authorized key and sudo access are configured. It applies the SSH public-key policy for adam and preserves the platform's existing management ports and other logins.
+`config/adam` grants passwordless sudo. `repository/chown` gives adam ownership of the tracked checkout directly under `/srv` and its Git metadata.
+
+Before moving SSH to port 2222, allow inbound TCP/2222 from the operator’s address through the provider firewall and the node’s firewall (UFW on stock Debian or the PVE firewall). Keep the original administrator session open throughout the transition.
+
+1. Run `./commons/apex config/hardening begin` from the node checkout in the original SSH session.
+2. On your workstation, run `ssh -o ControlPath=none -o BatchMode=yes -p 2222 adam@node.example.com` using the operator key. In the new session, run `./commons/apex config/hardening finalize` from the checkout.
+3. Run the same SSH command again from your workstation. From the checkout in this second fresh session, run `./commons/apex config/hardening commit` to cancel automatic rollback.
+
+On stock Debian the final policy allows only `adam` on port 2222 using a public key. PVE retains ports 22 and 2222, requires public keys globally, and keeps root key access with `PermitRootLogin prohibit-password` and no `AllowUsers` restriction.
+
+If a login fails, run `./commons/apex config/hardening rollback` from the checkout in the original session or let the timer restore the original SSH configuration five minutes after `begin`. Use `./commons/apex config/hardening status` to inspect the transition. [Action reference](https://docs.ermnvldmr.com/en/apex/reference/actions/#confighardening)
 
 ### Infrastructure Administrator
 

actions/config/hardening.py

@@ -1,42 +1,39 @@
-"""Apply the image's public-key SSH policy to adam after a verified key login."""
+"""Hand SSH access over to fresh, verified adam sessions on port 2222."""
 import ipaddress
 import json
 import os
+import platform
 import pwd
 import re
+import time
+import uuid
 from pathlib import Path
-from engine.descriptor import Meta
+from engine.descriptor import Arg, Meta, Opt
 
-METADATA = Meta(summary="Require SSH public-key authentication for adam from an independent administrator or verified adam key session.")
-CONFIG = "/etc/ssh/sshd_config"
-POLICY = """
-# APEX operator SSH policy
-Match User adam
-    AuthenticationMethods publickey
-    PubkeyAuthentication yes
-    PasswordAuthentication no
-    KbdInteractiveAuthentication no
-    PermitEmptyPasswords no
-    X11Forwarding no
-"""
+METADATA = Meta(summary="Manage the protected SSH handover to port 2222.", args=[
+    Arg("operation", "begin (default), finalize, commit, rollback, or status", required=False),
+    Opt("--original-port", "Current SSH port when beginning from a console"),
+    Opt("--client-address", "SSH client IP when beginning from a console"),
+])
+STATE = "/var/lib/apex-init/access/manual-state.json"
+BUNDLES = Path("/usr/local/lib/apex-provisioner")
 
 
 def _ancestors():
     ancestors = {}
     pid = os.getppid()
     ticks = os.sysconf("SC_CLK_TCK")
+    # proc starttime includes suspend time, while journald monotonic timestamps do not.
+    suspend_us = (time.clock_gettime_ns(time.CLOCK_BOOTTIME) - time.monotonic_ns()) // 1000
     while pid > 1:
         # comm can contain spaces and parentheses; the remaining fields start at state.
         fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()
-        ancestors[pid] = int(fields[19]) * 1_000_000 // ticks
+        ancestors[pid] = int(fields[19]) * 1_000_000 // ticks - max(0, suspend_us)
         pid = int(fields[1])
     return ancestors
 
 
-def _connection(ctx):
-    if pwd.getpwuid(os.getuid()).pw_name != "adam":
-        raise RuntimeError("Run config/hardening inside an SSH public-key session as adam.")
-    ancestors = _ancestors()
+def _socket(ctx, ancestors):
     sockets = ctx.sys.sudo(["ss", "-Hntp", "state", "established"], capture=True).stdout
     connections = set()
     for line in sockets.splitlines():
@@ -44,6 +41,8 @@ def _connection(ctx):
         if len(fields) == 5 and any(int(pid) in ancestors for pid in
                                     re.findall(r'"sshd(?:-session)?",pid=(\d+),', fields[4])):
             connections.add((fields[2], fields[3]))
+    if not connections:
+        return None
     if len(connections) != 1:
         raise RuntimeError("Cannot identify a unique established SSH socket for this process.")
     local, peer = connections.pop()
@@ -53,6 +52,19 @@ def _connection(ctx):
         ip = ipaddress.ip_address(value.strip("[]"))
         return str(getattr(ip, "ipv4_mapped", None) or ip)
     peer, local = address(peer), address(local)
+    return local, int(local_port), peer, int(peer_port)
+
+
+def _connection(ctx, *, after=None):
+    if pwd.getpwuid(os.getuid()).pw_name != "adam":
+        raise RuntimeError("Run this step inside a fresh SSH public-key session as adam.")
+    ancestors = _ancestors()
+    endpoint = _socket(ctx, ancestors)
+    if not endpoint:
+        raise RuntimeError("Cannot identify the SSH connection for this process.")
+    local, local_port, peer, peer_port = endpoint
+    if after is not None and local_port != 2222:
+        raise RuntimeError("Open a fresh adam SSH connection on port 2222.")
     record = rf"^Accepted publickey for adam from {re.escape(peer)} port {int(peer_port)} ssh2:"
     result = ctx.sys.sudo(["journalctl", "-b", "_UID=0", "_EXE=/usr/sbin/sshd",
                            "_EXE=/usr/lib/openssh/sshd-session", "_EXE=/usr/lib/openssh/sshd-auth",
@@ -62,10 +74,11 @@ def _connection(ctx):
             entry = json.loads(line)
             pid = int(entry["_PID"])
             if (pid in ancestors and int(entry["__MONOTONIC_TIMESTAMP"]) >= ancestors[pid]
+                    and (after is None or ancestors[pid] > after)
                     and re.match(record, entry["MESSAGE"])):
                 ctx.sys.run(["sudo", "-n", "true"])
                 return f"user=adam,host={peer},addr={peer},laddr={local},lport={int(local_port)}"
-    raise RuntimeError("Cannot verify public-key login for this SSH process in the root sshd journal.")
+    raise RuntimeError("Cannot verify a fresh public-key login for this SSH process in the root sshd journal.")
 
 
 def _operator_access(ctx):
@@ -75,7 +88,7 @@ def _operator_access(ctx):
     if pwd.getpwuid(os.getuid()).pw_name == "adam":
         return _connection(ctx)
     loginuid = int(Path("/proc/self/loginuid").read_text())
-    if loginuid in (account.pw_uid, 4294967295) or loginuid < 0:
+    if loginuid == account.pw_uid or (loginuid == 4294967295 and os.getuid() != 0) or loginuid < 0:
         raise RuntimeError("Use an independent administrator login or an SSH public-key session as adam.")
     if os.getuid() not in (0, loginuid):
         raise RuntimeError("Run from the independent administrator's own login.")
@@ -112,32 +125,65 @@ def _effective(system, connection):
     return dict(line.split(" ", 1) for line in result.stdout.splitlines() if " " in line)
 
 
+def _state(system):
+    if system.sudo(["test", "-f", STATE], check=False).returncode:
+        return {}
+    return json.loads(system.sudo(["cat", STATE], capture=True).stdout)
+
+
+def _original_endpoint(ctx, args):
+    endpoint = _socket(ctx, _ancestors())
+    port = getattr(args, "original_port", None)
+    address = getattr(args, "client_address", None)
+    if endpoint:
+        if port or address:
+            raise ValueError("SSH sessions detect the peer address and port automatically.")
+        return endpoint[1], endpoint[2]
+    if not port or not address or not 1 <= int(port) <= 65535:
+        raise ValueError("From a console, supply --original-port and --client-address for the SSH client.")
+    return int(port), str(ipaddress.ip_address(address))
+
+
 def run(ctx, args):
-    connection = _operator_access(ctx)
-    system = ctx.sys
-    system.sudo(["/usr/sbin/sshd", "-t"])
-    effective = _effective(system, connection)
-    if effective.get("authenticationmethods") not in ("any", "publickey"):
-        raise RuntimeError("Preserve the existing SSH authentication chain; reconcile it explicitly.")
-    required = {"authenticationmethods": "publickey", "pubkeyauthentication": "yes",
-                "passwordauthentication": "no", "kbdinteractiveauthentication": "no",
-                "permitemptypasswords": "no", "x11forwarding": "no"}
-    if all(effective.get(key) == value for key, value in required.items()):
+    operation = getattr(args, "operation", None) or "begin"
+    if operation not in ("begin", "finalize", "commit", "rollback", "status"):
+        raise ValueError("Choose begin, finalize, commit, rollback, or status.")
+    if operation == "begin":
+        _operator_access(ctx)
+    state = _state(ctx.sys)
+    phase = state.get("status", "absent")
+    if operation == "status" and phase == "absent":
+        print("absent")
         return
-    if Path(CONFIG).is_symlink():
-        raise RuntimeError(f"Refusing redirected SSH configuration: {CONFIG}")
-    original = system.sudo(["cat", CONFIG], capture=True).stdout
-    if "# APEX operator SSH policy" in original:
-        raise RuntimeError("The managed SSH policy is ineffective; reconcile earlier directives.")
-    ctx.host.write_file(CONFIG, original + "\n" + POLICY, sudo=True)
-    try:
-        system.sudo(["/usr/sbin/sshd", "-t"])
-        effective = _effective(system, connection)
-        if any(effective.get(key) != value for key, value in required.items()):
-            raise RuntimeError("The effective SSH policy differs; inspect existing Match blocks.")
-        system.reload("ssh")
-    except BaseException:
-        ctx.host.write_file(CONFIG, original, backup=False, sudo=True)
-        system.sudo(["/usr/sbin/sshd", "-t"])
-        system.reload("ssh")
-        raise
+    if operation == "begin" and phase in ("transition", "final", "recovery_pending"):
+        raise RuntimeError(f"Handover is {phase}; inspect status, continue from a fresh SSH login, or roll back.")
+    if operation in ("finalize", "commit"):
+        required = "transition" if operation == "finalize" else "final"
+        if phase != required:
+            raise RuntimeError(f"{operation} requires phase {required}; current phase is {phase}.")
+        _connection(ctx, after=state["authenticated_after_us"])
+    profile = state.get("profile") if operation != "begin" else None
+    if not profile:
+        if Path("/usr/bin/pveversion").is_file():
+            profile = "proxmox"
+        elif platform.freedesktop_os_release().get("ID") == "debian":
+            profile = "debian"
+        else:
+            raise RuntimeError("SSH hardening supports Debian and Proxmox VE.")
+    options = ["--profile", profile]
+    if operation == "begin":
+        port, address = _original_endpoint(ctx, args)
+        options.extend(["--original-port", str(port), "--client-address", address, "--token", uuid.uuid4().hex])
+        installer = Path(__file__).resolve().parents[2] / "ci/scripts/init-node/provisioning/manual_bundle.py"
+        bundle = ctx.sys.sudo(["/usr/bin/python3", "-I", "-B", str(installer)], capture=True).stdout.strip()
+    else:
+        if not state:
+            raise RuntimeError("No SSH handover has started.")
+        bundle = state["bundle"]
+        options.extend(["--token", state["token"]])
+    if not re.fullmatch(re.escape(str(BUNDLES)) + "/[0-9a-f]{64}", bundle):
+        raise RuntimeError("Invalid installed rollback bundle location.")
+    command = ["/usr/bin/python3", "-I", "-c",
+               "import sys; sys.path.insert(0, sys.argv.pop(1)); from provisioning.access import main; sys.exit(main())",
+               bundle, "tick" if operation == "status" else operation, *options]
+    ctx.sys.sudo(command)

ci/scripts/init-node/provisioning/access.py

@@ -54,8 +54,21 @@ def include_arguments(value):
     return arguments
 
 
+MANUAL_PROFILES = ('debian', 'proxmox')
+OPERATOR_MATCH = """Match User adam
+AuthenticationMethods publickey
+PubkeyAuthentication yes
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+PermitEmptyPasswords no
+X11Forwarding no""".splitlines()
+
+
 class Host:
-    def __init__(self, root=Path('/'), runner=subprocess.run):
+    def __init__(self, root=Path('/'), runner=subprocess.run, profile=None):
+        if profile is not None and profile not in MANUAL_PROFILES:
+            raise AccessError('Unknown SSH access profile')
+        self.profile = profile
         self.root = Path(root).resolve()
         self.runner = runner
 
@@ -136,7 +149,16 @@ class Host:
             remaining -= len(content)
             if remaining < 0:
                 raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required')
-            for line in content.decode().splitlines():
+            lines = content.decode().splitlines()
+            if self.profile and path == self.path('/etc/ssh/sshd_config'):
+                # The legacy action appended this exact terminal block. Any additional
+                # directives would change its meaning and require manual inspection.
+                marker = '# APEX operator SSH policy'
+                if marker in lines:
+                    position = lines.index(marker)
+                    if [item.strip() for item in lines[position + 1:] if item.strip()] == OPERATOR_MATCH:
+                        lines = lines[:position]
+            for line in lines:
                 line = line.lstrip()
                 if not line or line.startswith('#'):
                     continue
@@ -236,6 +258,10 @@ class Host:
     def verify_active(self, ports, final, address):
         args, pid = self.live_arguments()
         self.verify_policy([arg for arg in args if arg != '-D'], ports, final, address)
+        if self.listener_ports(pid) != set(ports):
+            raise AccessError('Active SSH listening ports differ from the declared policy')
+
+    def listener_ports(self, pid):
         output = self.command(['ss', '-H', '-ltnp']).stdout
         actual_ports = set()
         for line in output.splitlines():
@@ -247,19 +273,41 @@ class Host:
             if len(fields) < 5 or not fields[3].rsplit(':', 1)[-1].isdecimal():
                 raise AccessError('Unable to verify the active SSH listening sockets')
             actual_ports.add(int(fields[3].rsplit(':', 1)[-1]))
-        if actual_ports != set(ports):
-            raise AccessError('Active SSH listening ports differ from the declared policy')
+        return actual_ports
+
+    @property
+    def final_ports(self):
+        return [22, 2222] if self.profile == 'proxmox' else [2222]
+
+    def authentication_options(self):
+        proxmox = self.profile == 'proxmox'
+        options = {'PermitRootLogin': 'prohibit-password' if proxmox else 'no',
+                   'PasswordAuthentication': 'no', 'KbdInteractiveAuthentication': 'no',
+                   'PubkeyAuthentication': 'yes', 'AuthenticationMethods': 'publickey'}
+        if not proxmox:
+            options['AllowUsers'] = 'adam'
+        options.update(X11Forwarding='no', PermitEmptyPasswords='no')
+        return options
 
     def policy(self, ports, final, address):
         args = ['/usr/sbin/sshd']
         for port in sorted(set(ports)):
             args += ['-p', str(port)]
         if final:
-            for value in ('PermitRootLogin=no', 'PasswordAuthentication=no',
-                          'KbdInteractiveAuthentication=no', 'PubkeyAuthentication=yes',
-                          'AuthenticationMethods=publickey', 'AllowUsers=adam',
-                          'X11Forwarding=no', 'PermitEmptyPasswords=no'):
-                args += ['-o', value]
+            for key, value in self.authentication_options().items():
+                args += ['-o', f'{key}={value}']
+        elif self.profile:
+            # Keep existing managed authentication options when adding the probe port.
+            current = iter(self.service_arguments()[1:])
+            for flag in current:
+                if flag == '-D':
+                    continue
+                value = next(current, '')
+                if flag == '-p' and value.isdecimal():
+                    continue
+                if flag != '-o' or not re.fullmatch(r'[A-Za-z]+=[A-Za-z0-9,-]+', value):
+                    raise AccessError('Cannot safely extend the existing SSH invocation')
+                args += [flag, value]
         self.verify_policy(args, ports, final, address)
         return args
 
@@ -277,12 +325,15 @@ class Host:
                 values.setdefault(key, []).append(value)
             if set(values.get('port', [])) != {str(p) for p in ports}:
                 raise AccessError('Effective SSH ports do not match the handover policy')
-            if final and any(values.get(k) != [v] for k, v in {
-                    'permitrootlogin': 'no', 'passwordauthentication': 'no',
-                    'kbdinteractiveauthentication': 'no', 'pubkeyauthentication': 'yes',
-                    'authenticationmethods': 'publickey', 'allowusers': 'adam',
-                    'x11forwarding': 'no', 'permitemptypasswords': 'no'}.items()):
-                raise AccessError('Effective SSH authentication policy is inconsistent')
+            if final:
+                expected = {key.lower(): [value] for key, value in self.authentication_options().items()}
+                if self.profile == 'proxmox':
+                    if values.get('permitrootlogin') == ['without-password']:
+                        values['permitrootlogin'] = ['prohibit-password']
+                    if any(key in values for key in ('allowusers', 'denyusers', 'allowgroups', 'denygroups')):
+                        raise AccessError('SSH account restrictions conflict with Proxmox root-key access')
+                if any(values.get(key) != value for key, value in expected.items()):
+                    raise AccessError('Effective SSH authentication policy is inconsistent')
 
     def current_final(self, address):
         try:
@@ -374,19 +425,27 @@ def atomic(path, data, mode=0o600):
 
 
 class Transition:
-    def __init__(self, host=None, now=time.monotonic, security=None):
+    def __init__(self, host=None, now=time.monotonic, security=None, profile=None):
         from .security import Security
+        if profile is not None and profile not in MANUAL_PROFILES:
+            raise AccessError('Unknown SSH access profile')
+        self.profile = profile
         self.host = host or Host()
+        self.host.profile = profile
         self.security = security if security is not None else Security(self.host)
         self.now = now
         self.directory = self.host.path('/var/lib/apex-init/access')
 
     def load(self):
-        path = self.directory / 'state.json'
+        return self._read_record(bool(self.profile))
+
+    def _read_record(self, manual):
+        path = self.directory / ('manual-state.json' if manual else 'state.json')
         return json.loads(path.read_text()) if path.exists() else None
 
     def save(self, state):
-        atomic(self.directory / 'state.json', json.dumps(state, sort_keys=True).encode())
+        path = self.directory / ('manual-state.json' if state.get('profile') else 'state.json')
+        atomic(path, json.dumps(state, sort_keys=True).encode())
 
     def execute(self, operation, token=None, original_port=None, client_address=None):
         # Rollback must never wait for a stalled enrollment stage holding init.lock.
@@ -413,6 +472,24 @@ class Transition:
         with os.fdopen(descriptor, 'w') as lock:
             fcntl.flock(lock, fcntl.LOCK_EX)
             state = self.load()
+            other = self._read_record(not self.profile)
+            active = ('transition', 'final', 'recovery_pending')
+            if other and other['status'] in active:
+                if operation == 'tick':
+                    state = other
+                else:
+                    raise AccessError('Another access workflow is pending; wait for rollback before retrying')
+            if state and self.profile and state.get('profile') != self.profile and operation != 'tick':
+                if operation != 'begin' or state['status'] in active:
+                    raise AccessError('Access transition belongs to a different platform')
+                state = None
+            if operation == 'tick' and other and other.get('disarm_pending') and not (state and state['status'] in active):
+                state = other
+            if state and state.get('disarm_pending') and state['status'] in ('committed', 'rolled_back'):
+                self._disarm(state)
+            if (state and token and token == state.get('token')
+                    and (operation, state['status']) in (('commit', 'committed'), ('rollback', 'rolled_back'))):
+                return {'status': state['status']}
             if state and state['status'] == 'recovery_pending':
                 self._rollback(state)
                 if operation != 'tick':
@@ -437,6 +514,8 @@ class Transition:
                     raise AccessError('Invalid SSH client address') from None
                 self.host.preflight()
                 path = self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+                if self.profile:
+                    return self._begin_manual(token, original_port, client_address, path)
                 preserve_policy = False
                 if state and state['status'] == 'committed':
                     if state['fingerprint'] != self.host.fingerprint():
@@ -484,8 +563,14 @@ class Transition:
                     raise AccessError('Access transition is not ready for final policy')
                 try:
                     if not state.get('preserve_policy'):
-                        self.host.install_policy([2222], True, state['client_address'])
-                    self.security.finalize(state['security'], lambda value: self._save_security(state, value))
+                        if state.get('profile'):
+                            state['ssh_changed'] = True
+                            self.save(state)
+                        self.host.install_policy(self.host.final_ports, True, state['client_address'])
+                    if state.get('profile'):
+                        state['authenticated_after_us'] = time.monotonic_ns() // 1000
+                    else:
+                        self.security.finalize(state['security'], lambda value: self._save_security(state, value))
                     state['status'] = 'final'
                     state['fingerprint'] = self.host.fingerprint()
                     self.save(state)
@@ -494,14 +579,47 @@ class Transition:
                     raise
                 return {'status': 'final'}
             if operation == 'commit' and state['status'] == 'final':
-                self.host.verify_active([2222], True, state['client_address'])
-                self.security.verify(state['security'])
+                self.host.verify_active(self.host.final_ports, True, state['client_address'])
+                if not state.get('profile'):
+                    self.security.verify(state['security'])
                 state['status'] = 'committed'
+                state['disarm_pending'] = True
                 self.save(state)  # Timer observes this before it can restore anything.
-                self.host.disarm()
+                self._disarm(state)
                 return {'status': 'committed'}
             raise AccessError('Invalid access transition operation')
 
+    def _begin_manual(self, token, original_port, address, path):
+        _, pid = self.host.live_arguments()
+        ports = self.host.listener_ports(pid)
+        if original_port not in ports:
+            raise AccessError('Original SSH port is not active on the managed daemon')
+        preserve_policy = False
+        if ports == set(self.host.final_ports):
+            try:
+                self.host.verify_active(self.host.final_ports, True, address)
+                preserve_policy = True
+            except AccessError:
+                pass
+        previous = base64.b64encode(path.read_bytes()).decode() if path.exists() else None
+        state = {'status': 'transition', 'profile': self.profile, 'token': token,
+                 'deadline': self.now() + 300, 'boot_id': self.host.boot_id(),
+                 'bundle': str(Path(__file__).resolve().parents[1]),
+                 'client_address': address, 'original_port': original_port, 'previous': previous,
+                 'preserve_policy': preserve_policy, 'ssh_changed': 2222 not in ports}
+        self.host.arm()
+        self.save(state)
+        try:
+            if 2222 not in ports:
+                self.host.install_policy(sorted(ports | {2222}), False, address)
+            state['fingerprint'] = self.host.fingerprint()
+            state['authenticated_after_us'] = time.monotonic_ns() // 1000
+            self.save(state)
+        except Exception:
+            self._rollback(state)
+            raise
+        return {'status': 'transition'}
+
     def _save_security(self, state, value):
         state['security'] = value
         self.save(state)
@@ -513,7 +631,7 @@ class Transition:
                 self.security.restore(state['security'], lambda value: self._save_security(state, value))
             except Exception:
                 errors.append('security')
-        if not state.get('ssh_restored'):
+        if not state.get('ssh_restored') and state.get('ssh_changed', True):
             try:
                 self.host.restore(state)
                 state['ssh_restored'] = True
@@ -527,14 +645,21 @@ class Transition:
                               '; watchdog remains armed; inspect the root-owned access record')
         state['status'] = 'rolled_back'
         state.pop('recovery_errors', None)
+        state['disarm_pending'] = True
         self.save(state)
+        self._disarm(state)
+
+    def _disarm(self, state):
         self.host.disarm()
+        state.pop('disarm_pending', None)
+        self.save(state)
 
 
 def main(argv=None):
     from .security import SecurityError
     parser = argparse.ArgumentParser()
     parser.add_argument('operation', choices=['preflight', 'begin', 'finalize', 'commit', 'rollback', 'tick'])
+    parser.add_argument('--profile', choices=MANUAL_PROFILES)
     parser.add_argument('--token')
     parser.add_argument('--original-port', type=int)
     parser.add_argument('--client-address')
@@ -543,10 +668,10 @@ def main(argv=None):
         parser.error('Access handover requires root')
     try:
         if args.operation == 'preflight':
-            Host().bootstrap_preflight()
+            Host(profile=args.profile).bootstrap_preflight()
             result = {'status': 'prerequisites-verified'}
         else:
-            result = Transition().execute(args.operation, args.token, args.original_port, args.client_address)
+            result = Transition(profile=args.profile).execute(args.operation, args.token, args.original_port, args.client_address)
         print(json.dumps(result))
         return 0
     except (AccessError, SecurityError) as exc:

ci/scripts/init-node/provisioning/manual_bundle.py

@@ -0,0 +1,61 @@
+"""Install an immutable watchdog bundle from the operator's checkout."""
+import io
+import os
+from pathlib import Path
+import shutil
+import stat
+import sys
+import tarfile
+import tempfile
+
+
+def _trusted(path):
+    info = path.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o022:
+        raise RuntimeError(f"Untrusted provisioning directory: {path}")
+
+
+def install(root=Path('/')):
+    source = Path(__file__).resolve().parents[1]
+    sys.path.insert(0, str(source))
+    from provisioning.bundle import _bundle_from_source
+
+    data, digest = _bundle_from_source(source)
+    # All ancestors from the filesystem root must be protected before publishing code.
+    parent = root
+    _trusted(parent)
+    for component in ('usr', 'local', 'lib', 'apex-provisioner'):
+        parent = parent / component
+        parent.mkdir(mode=0o755, exist_ok=True)
+        _trusted(parent)
+    destination = parent / digest
+    with tempfile.TemporaryDirectory(prefix='.install-', dir=parent) as temporary:
+        stage = Path(temporary) / 'bundle'
+        stage.mkdir(mode=0o755)
+        with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+            for member in archive:
+                if not member.isfile() or '..' in Path(member.name).parts or Path(member.name).is_absolute():
+                    raise RuntimeError('Invalid provisioning bundle member')
+                target = stage / member.name
+                target.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+                with archive.extractfile(member) as source_file, target.open('xb') as output:
+                    shutil.copyfileobj(source_file, output)
+                target.chmod(member.mode)
+        try:
+            os.rename(stage, destination)
+        except OSError:
+            _trusted(destination)
+            installed, existing_digest = _bundle_from_source(destination)
+            if existing_digest != digest or installed != data:
+                raise RuntimeError('Installed provisioning bundle differs from the checkout') from None
+            for path in destination.rglob('*'):
+                info = path.lstat()
+                if info.st_uid != os.geteuid() or info.st_mode & 0o022 or path.is_symlink():
+                    raise RuntimeError(f'Untrusted installed provisioning file: {path}') from None
+    return destination
+
+
+if __name__ == '__main__':
+    if os.geteuid() != 0:
+        raise SystemExit('Run bundle installation as root.')
+    print(install())

ci/scripts/init-node/tests/test_provision_access.py

@@ -192,6 +192,63 @@ class AccessTests(unittest.TestCase):
         self.assertEqual(self.transition.load()['status'], 'committed')
         self.assertFalse(self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf').exists())
 
+    def manual(self, profile):
+        self.host.live_arguments = lambda: (['/usr/sbin/sshd', '-D'], '755')
+        self.host.listener_ports = lambda pid: set(self.host.ports)
+        return Transition(self.host, now=lambda: self.clock, profile=profile)
+
+    def test_manual_debian_keeps_all_old_ports_then_limits_to_operator_port(self):
+        self.host.ports = [22, 2200]
+        manual = self.manual('debian')
+        manual.execute('begin', TOKEN, 2200, '192.0.2.1')
+        self.assertEqual(self.host.ports, [22, 2200, 2222])
+        boundary = manual.load()['authenticated_after_us']
+        with self.assertRaisesRegex(AccessError, 'workflow is pending'):
+            self.begin()
+        manual.execute('finalize', TOKEN)
+        self.assertEqual(self.host.ports, [2222])
+        self.assertGreaterEqual(manual.load()['authenticated_after_us'], boundary)
+        manual.execute('commit', TOKEN)
+        self.assertIsNone(self.transition.load())
+        self.assertNotIn('security', manual.load())
+
+    def test_manual_proxmox_installed_policy_never_restarts_and_preserves_image_state(self):
+        self.host.ports = [22, 2222]
+        self.transition.save({'status': 'committed', 'token': 'b' * 32})
+        manual = self.manual('proxmox')
+        with patch.object(self.host, 'install_policy', side_effect=AssertionError('restarted sshd')):
+            for round_number in range(2):
+                token = str(round_number) * 32
+                manual.execute('begin', token, 22, '192.0.2.1')
+                manual.execute('finalize', token)
+                manual.execute('commit', token)
+        self.assertEqual(self.host.ports, [22, 2222])
+        self.assertEqual(self.transition.load(), {'status': 'committed', 'token': 'b' * 32})
+
+    def test_plain_watchdog_restores_manual_override_on_reboot(self):
+        override = self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf')
+        atomic(override, b'original managed invocation')
+        manual = self.manual('proxmox')
+        manual.execute('begin', TOKEN, 22, '192.0.2.1')
+        manual.execute('finalize', TOKEN)
+        self.assertEqual(self.host.ports, [22, 2222])
+        self.host.boot = 'second-boot'
+        Transition(self.host, now=lambda: 0).execute('tick')
+        self.assertEqual(override.read_bytes(), b'original managed invocation')
+        self.assertEqual(manual.load()['status'], 'rolled_back')
+        self.assertFalse(self.host.armed)
+
+    def test_failed_disarm_is_retried_after_durable_commit_without_rollback(self):
+        self.begin()
+        self.transition.execute('finalize', TOKEN)
+        with patch.object(self.host, 'disarm', side_effect=AccessError('unavailable')), self.assertRaises(AccessError):
+            self.transition.execute('commit', TOKEN)
+        self.clock += 400
+        self.transition.execute('tick')
+        self.assertEqual(self.transition.load()['status'], 'committed')
+        self.assertFalse(self.host.armed)
+        self.assertEqual(self.host.restored, 0)
+
     def test_reject_symlink_state_ancestor(self):
         outside = self.host.path('/outside')
         outside.mkdir()
@@ -445,6 +502,35 @@ class OpenSSHPolicyTests(unittest.TestCase):
                 self.assertIn('x11forwarding no', effective)
                 self.assertIn('permitemptypasswords no', effective)
 
+    def test_proxmox_effective_root_key_auth_and_both_ports_with_legacy_operator_block(self):
+        self.host.profile = 'proxmox'
+        with self.config.open('a') as stream:
+            stream.write('# APEX operator SSH policy\nMatch User adam\n'
+                         '  AuthenticationMethods publickey\n  PubkeyAuthentication yes\n'
+                         '  PasswordAuthentication no\n  KbdInteractiveAuthentication no\n'
+                         '  PermitEmptyPasswords no\n  X11Forwarding no\n')
+        final = self.host.policy([22, 2222], True, '192.0.2.1')
+        for user in ('root', 'adam'):
+            effective = self.effective(final, user)
+            self.assertEqual({item for item in effective if item.startswith('port ')}, {'port 22', 'port 2222'})
+            self.assertTrue({'permitrootlogin without-password', 'permitrootlogin prohibit-password'}.intersection(effective))
+            self.assertIn('authenticationmethods publickey', effective)
+            self.assertFalse(any(item.startswith('allowusers ') for item in effective))
+        with self.config.open('a') as stream:
+            stream.write('  PasswordAuthentication yes\n')
+        with self.assertRaisesRegex(AccessError, 'Match'):
+            self.host.policy([22, 2222], True, '192.0.2.1')
+
+    def test_adding_probe_port_preserves_authentication_from_the_existing_service(self):
+        self.host.profile = 'debian'
+        args = ['/usr/sbin/sshd', '-p', '2200', '-o', 'PermitRootLogin=no', '-o', 'PasswordAuthentication=no', '-D']
+        with patch.object(self.host, 'service_arguments', return_value=args):
+            transition = self.host.policy([2200, 2222], False, '192.0.2.1')
+        effective = self.effective(transition, 'root')
+        self.assertIn('permitrootlogin no', effective)
+        self.assertIn('passwordauthentication no', effective)
+        self.assertIn('port 2200', effective)
+
     def test_additive_inherited_allowusers_stops_finalization(self):
         with self.config.open('a') as stream:
             stream.write('AllowUsers provider\n')

tests/test_manual_host_setup.py

@@ -2,6 +2,7 @@ import json
 import os
 from pathlib import Path
 import subprocess
+import sys
 import tempfile
 from types import SimpleNamespace
 import unittest
@@ -27,18 +28,75 @@ class HostSetup(unittest.TestCase):
         self.assertIn("sudo", requested)
         self.assertNotIn("ufw", requested)
 
-    def test_failed_effective_policy_restores_original_configuration(self):
+    def test_fresh_key_proof_gates_each_destructive_phase(self):
         hardening = action("config/hardening")
-        system, host = Mock(), Mock()
-        system.sudo.return_value.stdout = "Port 22\n"
-        with (patch.object(hardening, "_operator_access", return_value="verified"),
-              patch.object(hardening, "_effective", return_value={"authenticationmethods": "any"})):
-            with self.assertRaisesRegex(RuntimeError, "effective SSH"):
-                hardening.run(SimpleNamespace(sys=system, host=host), None)
-        writes = host.write_file.call_args_list
-        self.assertTrue(writes[0].args[1].startswith("Port 22\n"))
-        self.assertEqual(writes[-1].args[1], "Port 22\n")
-        system.reload.assert_called_once_with("ssh")
+        for phase, operation in (("transition", "finalize"), ("final", "commit")):
+            state = {"status": phase, "authenticated_after_us": 123,
+                     "bundle": "/usr/local/lib/apex-provisioner/" + "a" * 64, "token": "a" * 32}
+            system = Mock()
+            with (patch.object(hardening, "_state", return_value=state),
+                  patch.object(hardening, "_connection", side_effect=RuntimeError("fresh login required")) as proof):
+                with self.assertRaisesRegex(RuntimeError, "fresh login required"):
+                    hardening.run(SimpleNamespace(sys=system), SimpleNamespace(operation=operation))
+                proof.assert_called_once_with(unittest.mock.ANY, after=123)
+            self.assertFalse(system.mock_calls)
+
+    def test_begin_installs_bundle_before_dispatch_with_real_endpoint_and_profile(self):
+        hardening = action("config/hardening")
+        ctx = SimpleNamespace(sys=Mock())
+        bundle = "/usr/local/lib/apex-provisioner/" + "a" * 64
+        ctx.sys.sudo.return_value.stdout = bundle
+        with (patch.object(hardening, "_operator_access"),
+              patch.object(hardening, "_state", return_value={}),
+              patch.object(Path, "is_file", return_value=True),
+              patch.object(hardening, "_original_endpoint", return_value=(2022, "192.0.2.8"))):
+            hardening.run(ctx, None)
+        install, begin = [call.args[0] for call in ctx.sys.sudo.call_args_list]
+        self.assertTrue(install[-1].endswith("/manual_bundle.py"))
+        self.assertIn(bundle, begin)
+        self.assertEqual(begin[begin.index("begin"): -2],
+                         ["begin", "--profile", "proxmox", "--original-port", "2022", "--client-address", "192.0.2.8"])
+
+    def test_installed_watchdog_survives_checkout_and_dispatches_cli_arguments(self):
+        hardening = action("config/hardening")
+        root = Path(__file__).resolve().parents[1]
+        installer = overlay.load_module(str(root / "ci/scripts/init-node/provisioning/manual_bundle.py"),
+                                        "test_manual_bundle")
+        with tempfile.TemporaryDirectory() as temp, patch.object(sys, "path", list(sys.path)):
+            destination = installer.install(Path(temp).resolve())
+            self.assertEqual(installer.install(Path(temp).resolve()), destination)
+            state = {"status": "final", "authenticated_after_us": 120, "profile": "debian",
+                     "bundle": "/usr/local/lib/apex-provisioner/" + destination.name, "token": "a" * 32}
+            system = Mock()
+            with (patch.object(hardening, "_state", return_value=state),
+                  patch.object(hardening, "_connection")):
+                hardening.run(SimpleNamespace(sys=system), SimpleNamespace(operation="commit"))
+            # Run the generated CLI in isolation. Stop at its root check after argument parsing.
+            command = system.sudo.call_args.args[0]
+            command[0] = sys.executable
+            command[3] = "import os; os.geteuid = lambda: 1000; " + command[3]
+            command[4] = str(destination)
+            result = subprocess.run(command, cwd=temp, capture_output=True, text=True, timeout=30)
+            self.assertIn("Access handover requires root", result.stderr)
+            source = destination / "provisioning/access.py"
+            source.chmod(0o666)
+            with self.assertRaisesRegex(RuntimeError, "Untrusted installed"):
+                installer.install(Path(temp).resolve())
+            source.chmod(0o644)
+            source.write_text("changed")
+            with self.assertRaisesRegex(RuntimeError, "differs from the checkout"):
+                installer.install(Path(temp).resolve())
+
+    def test_other_linux_distributions_are_rejected_before_installing_the_watchdog(self):
+        hardening = action("config/hardening")
+        ctx = SimpleNamespace(sys=Mock())
+        with (patch.object(hardening, "_operator_access"),
+              patch.object(hardening, "_state", return_value={}),
+              patch.object(Path, "is_file", return_value=False),
+              patch.object(hardening.platform, "freedesktop_os_release", return_value={"ID": "ubuntu"})):
+            with self.assertRaisesRegex(RuntimeError, "supports Debian"):
+                hardening.run(ctx, None)
+        self.assertFalse(ctx.sys.mock_calls)
 
     def test_current_endpoint_must_have_authenticated_with_a_key(self):
         hardening = action("config/hardening")
@@ -61,6 +119,22 @@ class HostSetup(unittest.TestCase):
             self.assertEqual(hardening._connection(ctx),
                              "user=adam,host=192.0.2.8,addr=192.0.2.8,laddr=192.0.2.9,lport=22")
 
+    def test_reused_transport_and_wrong_port_cannot_finalize(self):
+        hardening = action("config/hardening")
+        ctx = SimpleNamespace(sys=Mock())
+        entry = {"_PID": "60", "__MONOTONIC_TIMESTAMP": "100",
+                 "MESSAGE": "Accepted publickey for adam from 192.0.2.8 port 3002 ssh2: ED25519"}
+        ctx.sys.sudo.return_value = SimpleNamespace(returncode=0, stdout=json.dumps(entry))
+        with (patch.object(hardening.pwd, "getpwuid", return_value=SimpleNamespace(pw_name="adam")),
+              patch.object(hardening, "_ancestors", return_value={60: 50})):
+            for port, after in ((22, 49), (2222, 50), (2222, 99), (2222, 101)):
+                with (self.subTest(port=port, after=after),
+                      patch.object(hardening, "_socket", return_value=("192.0.2.9", port, "192.0.2.8", 3002)),
+                      self.assertRaises(RuntimeError)):
+                    hardening._connection(ctx, after=after)
+            with patch.object(hardening, "_socket", return_value=("192.0.2.9", 2222, "192.0.2.8", 3002)):
+                self.assertIn("lport=2222", hardening._connection(ctx, after=49))
+
     def test_journald_restart_failure_restores_previous_config(self):
         base = action("config/base")
         with tempfile.TemporaryDirectory() as temporary: